From c95c5aff3f975aff8fe525096264b3ee7b85c2e1 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:39:57 -0300 Subject: [PATCH 01/19] clean release.yaml --- release.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/release.yaml b/release.yaml index 39aa552c47..8b13789179 100644 --- a/release.yaml +++ b/release.yaml @@ -1,8 +1 @@ -rancher-logging: - - 105.2.1+up4.10.0 -rancher-logging-crd: - - 105.2.1+up4.10.0 -rancher-monitoring: - - 105.1.3+up61.3.2 -rancher-monitoring-crd: - - 105.1.3+up61.3.2 + From 884038f5b7aa9f0d79f948f23c0cc537554061ad Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:41:55 -0300 Subject: [PATCH 02/19] fp: rancher-webhook 104.0.7+up0.5.7 --- .../rancher-webhook-104.0.7+up0.5.7.tgz | Bin 0 -> 2794 bytes .../104.0.7+up0.5.7/Chart.yaml | 14 +++ .../104.0.7+up0.5.7/templates/_helpers.tpl | 22 +++++ .../104.0.7+up0.5.7/templates/deployment.yaml | 82 ++++++++++++++++++ .../104.0.7+up0.5.7/templates/rbac.yaml | 12 +++ .../104.0.7+up0.5.7/templates/secret.yaml | 11 +++ .../104.0.7+up0.5.7/templates/service.yaml | 13 +++ .../templates/serviceaccount.yaml | 11 +++ .../104.0.7+up0.5.7/templates/webhook.yaml | 9 ++ .../104.0.7+up0.5.7/tests/README.md | 16 ++++ .../tests/deployment_test.yaml | 73 ++++++++++++++++ .../104.0.7+up0.5.7/tests/service_test.yaml | 18 ++++ .../104.0.7+up0.5.7/values.yaml | 30 +++++++ index.yaml | 18 ++++ release.yaml | 3 +- 15 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 assets/rancher-webhook/rancher-webhook-104.0.7+up0.5.7.tgz create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/Chart.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/_helpers.tpl create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/deployment.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/rbac.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/secret.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/service.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/serviceaccount.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/templates/webhook.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/tests/README.md create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/tests/deployment_test.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/tests/service_test.yaml create mode 100644 charts/rancher-webhook/104.0.7+up0.5.7/values.yaml diff --git a/assets/rancher-webhook/rancher-webhook-104.0.7+up0.5.7.tgz b/assets/rancher-webhook/rancher-webhook-104.0.7+up0.5.7.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ed805d0bea1e3c9b791699ee075caeb3f9f0be2d GIT binary patch literal 2794 zcmVDc zVQyr3R8em|NM&qo0PH(qZ`(H0{j6VcaNZaF@K&Vu0@v%EzkW_gq z60;Xi2EOn6!@5fB%7la%fNgZ;N-B(7k%bI^QJ($T+M8e+QzhLJ>(wpg zk+_wu*o8rqg1-F!AE58O){?^a(D!=2{`Y13U}Q)nx?1{jy$aiMNJ2jpRF{2+BuV+{ zdnewh6QK-KnkdusfiM~oMfrRb#Z+p%Uo0V*i4-m^aZsxSrPI!^oI}qac)oY~e_3J~ zt@poa|4YI$luzpbcG!RKqhR zL;z;+MaI~LU^H9>@OCy4>LNvnTsh91i78>ZTT#rg0kC%b24^H=>JsNvD$TQ_oyI~) zH9>VN(vJbm2$SeIaTx1?$jOvxFQgre;~aq!3{&g5j6tXcLM|vom@u)5kt+|b5MiNI zB7?4RK`BRtQd?^zLfH)k4>3`)E5ijw8KZpe;*y4%a90t^l}mW!uDNv$w^*bIl+T1y zkt2X#e{;Bq@Cq4*N~8hY-8;_F5nQBHq*SfOj7T{lF#?g2&N)V)L{(fTX=0kCagHDn zsme!5xLwe2VVjhIOd{MakVBerPWgPp5op8!UJnMxn$iVHNsJ0pyKp2^3*?H1#@U#4 zG_IN?RARcs)N?e7-h?9mxszP$*2 zT77ysEJ-O>GwA%U>~_A205XZ?M`tY$@@V6Kr#hpCc6*FWkx5H{B*|vPR8VRXGI67; zU3=vu$O*ad5%mF9jYqXhGNDwEkJSx0n_;K|=uE_wJ#0**ED}vm9~PanPZ?pI?enRK zA@d`~%1}Sp*70zKQv3dIxBtqa^vGy%hy4$R{%OPhPX~VgVE_97tswuS4_*j!fS(Kb z#sPy8|0XeO$M_8G6l;Rk;^-}M)_iYO)Ztc9t7$^3q*a7c(BT7uhJx$70Mm*>AYZO` zz0i-V>(Ti|MHh+#;qI>BEftBA`}@wu(6hI2Luvgi_L_13)o zf~ZAMny*)0_>{Ocqg;pA&?3|5-Mja{oV~djPp&p6`tt4${t%Qy=T)bT+RGZp+E(W& zuBB11%gS~ZT4fprKMzbbY>aJ@BKi54YKw>nY(_J88YVlX6-f1{@jAZp5 zov53%_+Qu87gyE38KEpo@p_S>TnH8gF!U?k38qv;R~QN&$pB8uE=HHgQOai$d#hYK zj-}#YIu{vl+ttnU2_V+E*7Uk6PP!kzlZB4CHc9Tb4NH+k?>Sr5&YD((X}I5zBokCW zQXXAlx}+i2v<1V=+;s@9@Xy*Zha@3W%BZ3!>&s7~sNUehpy$+KHOsz4Gjcb`T&9)nxigRuDuFbI`-Lu7i+vERqO2VhP2i}$c4~~1y z{J+=t5Apv#pc>Pg9oN1_r1UQn5qTfKGYMrG=#N<@6{eShp|)wt=U));r-ES~1(1aP zoQf=YfSCiZ!K)-4+BTO&G3AcTrhj0lU7wqexvJUsR*7tT{=Rf~=Uv)&8*fsyZcCT{ z>tMV6+sx%DJispdKklD2?f=9-*#BPO@8thm!;#vti^{^ zgOIIn!D^LY0Php>DMRO5I8Fa~@R0o*SNa4FV5j{LoA1BF;mP6tZ!b_gHk*dL`)Gf_ zIE(wWGUashM?F$XZ-Is-v9c3zdtzG|308$@vTwxZa3h%Zqs*s9VJ(0kbDY>A%Zyf=p}^aA)P zV+_ItNfnIWLYi^7rD_4hyr2}9bB^=-@4rupTsU6W9@VmI#>j3+L3frU`vF{|RGN|m zGAEj^sl2m11ZOK)lRGkU5ylM40$B_R%jT5BjHXh7TWc$Di`vgQRf}xug(B{1G&kc^ z>G{^byg5~l=BC%fEyS;*$c3H6bMM_#K)d~~YkzLcj2{;R?6Cji)4_4G{%7bP>i_lv zGNTG}E38!UlwqJv+#9uEj$_os_E8mpT!@TCAW>CHuxVSAGRQ z>+%gPP{(OoyDW)J-cqb#QcOs$-=OUsO8>a``gxYK9sE&m6MEp&FVE?vRikTH^ng}`QYTNNnjCNl{}oH?8*W5j1|s z;D%V5SfehfTFp`a5ksZp5AhVcUtg~s*4S1Y(pL8fJ8B;vrJ=QLdW6_tSq!tK9A-c6 zu)Wl+bcc0ad8IYv!Sla8{>$%vpCb7Cg8c8~c%%N?A0F<1_5#)TuXqJ&1b@|sz`aF& wWjL~aCE60&JrqdU!A~AXeUWg6^mA|T0S<7018jx=1ONd4|5}=`q5wny06pS?t^fc4 literal 0 HcmV?d00001 diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/Chart.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/Chart.yaml new file mode 100644 index 0000000000..ca632c1454 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/Chart.yaml @@ -0,0 +1,14 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: < 1.31.0-0 + catalog.cattle.io/namespace: cattle-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-webhook +apiVersion: v2 +appVersion: 0.5.7 +description: ValidatingAdmissionWebhook for Rancher types +name: rancher-webhook +version: 104.0.7+up0.5.7 diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/_helpers.tpl b/charts/rancher-webhook/104.0.7+up0.5.7/templates/_helpers.tpl new file mode 100644 index 0000000000..c37a65c6f3 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "rancher-webhook.labels" -}} +app: rancher-webhook +{{- end }} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/deployment.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/deployment.yaml new file mode 100644 index 0000000000..b8a7201dac --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/deployment.yaml @@ -0,0 +1,82 @@ +{{- $auth := .Values.auth | default dict }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rancher-webhook +spec: + selector: + matchLabels: + app: rancher-webhook + template: + metadata: + labels: + app: rancher-webhook + spec: + {{- if $auth.clientCA }} + volumes: + - name: client-ca + secret: + secretName: client-ca + {{- end }} + {{- if .Values.global.hostNetwork }} + hostNetwork: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 6 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 6 }} + {{- end }} + containers: + - env: + - name: STAMP + value: "{{.Values.stamp}}" + - name: ENABLE_MCM + value: "{{.Values.mcm.enabled}}" + - name: CATTLE_PORT + value: {{.Values.port | default 9443 | quote}} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $auth.allowedCNs }} + - name: ALLOWED_CNS + value: '{{ join "," $auth.allowedCNs }}' + {{- end }} + image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}' + name: rancher-webhook + imagePullPolicy: "{{ .Values.image.imagePullPolicy }}" + ports: + - name: https + containerPort: {{ .Values.port | default 9443 }} + startupProbe: + httpGet: + path: "/healthz" + port: "https" + scheme: "HTTPS" + failureThreshold: 60 + periodSeconds: 5 + livenessProbe: + httpGet: + path: "/healthz" + port: "https" + scheme: "HTTPS" + periodSeconds: 5 + {{- if $auth.clientCA }} + volumeMounts: + - name: client-ca + mountPath: /tmp/k8s-webhook-server/client-ca + readOnly: true + {{- end }} + {{- if .Values.capNetBindService }} + securityContext: + capabilities: + add: + - NET_BIND_SERVICE + {{- end }} + serviceAccountName: rancher-webhook + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/rbac.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/rbac.yaml new file mode 100644 index 0000000000..f4364995c0 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/rbac.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rancher-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: rancher-webhook + namespace: {{.Release.Namespace}} \ No newline at end of file diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/secret.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/secret.yaml new file mode 100644 index 0000000000..9fd331dc1e --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/secret.yaml @@ -0,0 +1,11 @@ +{{- $auth := .Values.auth | default dict }} +{{- if $auth.clientCA }} +apiVersion: v1 +data: + ca.crt: {{ $auth.clientCA }} +kind: Secret +metadata: + name: client-ca + namespace: cattle-system +type: Opaque +{{- end }} diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/service.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/service.yaml new file mode 100644 index 0000000000..220afebeae --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/service.yaml @@ -0,0 +1,13 @@ +kind: Service +apiVersion: v1 +metadata: + name: rancher-webhook + namespace: cattle-system +spec: + ports: + - port: 443 + targetPort: {{ .Values.port | default 9443 }} + protocol: TCP + name: https + selector: + app: rancher-webhook diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/serviceaccount.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/serviceaccount.yaml new file mode 100644 index 0000000000..9e7ad7e1fe --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rancher-webhook +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rancher-webhook-sudo + annotations: + cattle.io/description: "SA which can be impersonated to bypass rancher-webhook validation" \ No newline at end of file diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/templates/webhook.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/templates/webhook.yaml new file mode 100644 index 0000000000..53a0687b6f --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/templates/webhook.yaml @@ -0,0 +1,9 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: rancher.cattle.io +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: rancher.cattle.io diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/tests/README.md b/charts/rancher-webhook/104.0.7+up0.5.7/tests/README.md new file mode 100644 index 0000000000..6d3059a005 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/tests/README.md @@ -0,0 +1,16 @@ + +## local dev testing instructions + +Option 1: Full chart CI run with a live cluster + +```bash +./scripts/charts/ci +``` + +Option 2: Test runs against the chart only + +```bash +# install the helm plugin first - helm plugin install https://github.com/helm-unittest/helm-unittest.git +bash dev-scripts/helm-unittest.sh +``` + diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/tests/deployment_test.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/tests/deployment_test.yaml new file mode 100644 index 0000000000..bbd6e30444 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/tests/deployment_test.yaml @@ -0,0 +1,73 @@ +suite: Test Deployment +templates: + - deployment.yaml + +tests: + - it: should set webhook default port values + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 9443 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_PORT + value: "9443" + + - it: should set updated webhook port + set: + port: 2319 + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 2319 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CATTLE_PORT + value: "2319" + + - it: should not set capabilities by default. + asserts: + - isNull: + path: spec.template.spec.containers[0].securityContext + + - it: should set net capabilities when capNetBindService is true. + set: + capNetBindService: true + asserts: + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.add + content: NET_BIND_SERVICE + + - it: should not set volumes or volumeMounts by default + asserts: + - isNull: + path: spec.template.spec.volumes + - isNull: + path: spec.template.spec.volumeMounts + + - it: should set CA fields when CA options are set + set: + auth.clientCA: base64-encoded-cert + auth.allowedCNs: + - kube-apiserver + - joe + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: client-ca + secret: + secretName: client-ca + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: client-ca + mountPath: /tmp/k8s-webhook-server/client-ca + readOnly: true + - contains: + path: spec.template.spec.containers[0].env + content: + name: ALLOWED_CNS + value: kube-apiserver,joe diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/tests/service_test.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/tests/service_test.yaml new file mode 100644 index 0000000000..03172ad033 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/tests/service_test.yaml @@ -0,0 +1,18 @@ +suite: Test Service +templates: + - service.yaml + +tests: + - it: should set webhook default port values + asserts: + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: should set updated target port + set: + port: 2319 + asserts: + - equal: + path: spec.ports[0].targetPort + value: 2319 diff --git a/charts/rancher-webhook/104.0.7+up0.5.7/values.yaml b/charts/rancher-webhook/104.0.7+up0.5.7/values.yaml new file mode 100644 index 0000000000..d1b2854aa8 --- /dev/null +++ b/charts/rancher-webhook/104.0.7+up0.5.7/values.yaml @@ -0,0 +1,30 @@ +image: + repository: rancher/rancher-webhook + tag: v0.5.7 + imagePullPolicy: IfNotPresent + +global: + cattle: + systemDefaultRegistry: "" + hostNetwork: false + +mcm: + enabled: true + +# tolerations for the webhook deployment. See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ for more info +tolerations: [] +nodeSelector: {} + +## PriorityClassName assigned to deployment. +priorityClassName: "" + +# port assigns which port to use when running rancher-webhook +port: 9443 + +# Parameters for authenticating the kube-apiserver. +auth: + # CA for authenticating kube-apiserver client certs. If empty, client connections will not be authenticated. + # Must be base64-encoded. + clientCA: "" + # Allowlist of CNs for kube-apiserver client certs. If empty, any cert signed by the CA provided in clientCA will be accepted. + allowedCNs: [] diff --git a/index.yaml b/index.yaml index 849c8ae724..3342cf391a 100755 --- a/index.yaml +++ b/index.yaml @@ -22460,6 +22460,24 @@ entries: urls: - assets/rancher-webhook/rancher-webhook-105.0.0+up0.6.1.tgz version: 105.0.0+up0.6.1 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: < 1.31.0-0 + catalog.cattle.io/namespace: cattle-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-webhook + apiVersion: v2 + appVersion: 0.5.7 + created: "2025-02-25T21:41:15.57879279-03:00" + description: ValidatingAdmissionWebhook for Rancher types + digest: 6ec06c0a12861569268eb54e2f24fea91587a1e21cf545ceb15ad4e631c639c3 + name: rancher-webhook + urls: + - assets/rancher-webhook/rancher-webhook-104.0.7+up0.5.7.tgz + version: 104.0.7+up0.5.7 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index 8b13789179..8cdd2c58b2 100644 --- a/release.yaml +++ b/release.yaml @@ -1 +1,2 @@ - +rancher-webhook: + - 104.0.7+up0.5.7 From a9484c6674202aa45c91dfe4bbefeff29c0f74e3 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:42:37 -0300 Subject: [PATCH 03/19] fp: rancher-logging 104.2.0+up4.8.0 --- .../rancher-logging-104.2.0+up4.8.0.tgz | Bin 0 -> 19950 bytes .../104.2.0+up4.8.0/.helmignore | 26 + .../104.2.0+up4.8.0/Chart.yaml | 30 ++ .../rancher-logging/104.2.0+up4.8.0/README.md | 134 +++++ .../104.2.0+up4.8.0/app-readme.md | 45 ++ .../104.2.0+up4.8.0/templates/NOTES.txt | 0 .../templates/_generic_fluentbitagent.yaml | 85 +++ .../templates/_generic_logging.yaml | 75 +++ .../104.2.0+up4.8.0/templates/_helpers.tpl | 231 +++++++++ .../templates/clusterrole.yaml | 388 ++++++++++++++ .../templates/clusterrolebinding.yaml | 18 + .../104.2.0+up4.8.0/templates/crds.yaml | 6 + .../104.2.0+up4.8.0/templates/deployment.yaml | 79 +++ .../templates/extra-manifests.yaml | 4 + .../templates/logging/clusterflows.yaml | 14 + .../templates/logging/clusteroutputs.yaml | 14 + .../templates/logging/eventtailer.yaml | 41 ++ .../templates/logging/fluentbit.yaml | 17 + .../templates/logging/hosttailer.yaml | 31 ++ .../templates/logging/logging.yaml | 60 +++ .../loggings/aks/fluentbitagent.yaml | 20 + .../templates/loggings/aks/logging.yaml | 12 + .../loggings/eks/fluentbitagent.yaml | 21 + .../templates/loggings/eks/logging.yaml | 20 + .../loggings/gke/fluentbitagent.yaml | 20 + .../templates/loggings/gke/logging.yaml | 12 + .../templates/loggings/k3s/configmap.yaml | 57 +++ .../templates/loggings/k3s/daemonset.yaml | 112 ++++ .../loggings/k3s/fluentbitagent.yaml | 21 + .../loggings/k3s/logging-k3s-openrc.yaml | 12 + .../loggings/kube-audit/fluentbitagent.yaml | 24 + .../loggings/kube-audit/logging.yaml | 15 + .../templates/loggings/rke/configmap.yaml | 29 ++ .../templates/loggings/rke/daemonset.yaml | 124 +++++ .../templates/loggings/rke2/configmap.yaml | 69 +++ .../templates/loggings/rke2/daemonset.yaml | 118 +++++ .../loggings/root/fluentbitagent.yaml | 29 ++ .../templates/loggings/root/logging.yaml | 67 +++ .../104.2.0+up4.8.0/templates/service.yaml | 20 + .../templates/service_monitor.yaml | 30 ++ .../templates/serviceaccount.yaml | 14 + .../templates/test_receiver.yaml | 53 ++ .../104.2.0+up4.8.0/templates/userrole.yaml | 39 ++ .../104.2.0+up4.8.0/templates/userroles.yaml | 35 ++ .../templates/validate-install-crd.yaml | 29 ++ .../templates/validate-install.yaml | 5 + .../templates/validate-psp-install.yaml | 7 + .../values-logging-example.yaml | 24 + .../104.2.0+up4.8.0/values.yaml | 484 ++++++++++++++++++ index.yaml | 34 ++ release.yaml | 2 + 51 files changed, 2856 insertions(+) create mode 100644 assets/rancher-logging/rancher-logging-104.2.0+up4.8.0.tgz create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/.helmignore create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/Chart.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/README.md create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/app-readme.md create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/NOTES.txt create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/_helpers.tpl create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrole.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/crds.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/deployment.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/extra-manifests.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusterflows.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusteroutputs.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/eventtailer.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/fluentbit.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/hosttailer.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/logging/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/configmap.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/daemonset.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/logging-k3s-openrc.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/configmap.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/daemonset.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/configmap.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/daemonset.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/fluentbitagent.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/logging.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/service.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/service_monitor.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/serviceaccount.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/test_receiver.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/userrole.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/userroles.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install-crd.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/templates/validate-psp-install.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/values-logging-example.yaml create mode 100644 charts/rancher-logging/104.2.0+up4.8.0/values.yaml diff --git a/assets/rancher-logging/rancher-logging-104.2.0+up4.8.0.tgz b/assets/rancher-logging/rancher-logging-104.2.0+up4.8.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0d006aabf692ef2e3b966b0bc2ef81933e251ae5 GIT binary patch literal 19950 zcmV)qK$^cFiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ#avV3ZD181-KLt+H4wD*n^}PL3xlNWIHB<+ z(N*YfnAL?cfFji+`HTA+_x0|Re1XJLsI50HlE*|iEOr4%B(_9mA~V5p5>5&4#B4I5 z$)tNU#asq+Jd6M9)>5z6>mBUv!N0v;ul(t2aFODW43Ylmx+~^Fl38t~ zl<*mq0(ubBB>V7qQ9n}7u4qI=XL7}d$bb(1BcqY)NXFvYVj1m(Y?iWwB+_-U zzGNySCwP{3U@!PK=(TW4f3jb$1}&VXKNSl^M1-8C5IwVY)b1gbQ~CVA!f{5#?J@w1&;Q-Q;9$Re{_pqu59j}Vq%HJ}jByrAq-P$e zW_gPTtu1smr2TsheW^CqBA@piWA`^Wr9k^=Z^YyXOwfsN!0PFRi{Fb z5R!Oe&j^IlERK&^OvCvQy&S(`@|Y7r64`2z@oXg+vguO;|XF53q?0% zNk~#Dx+xH&t}wtmp@!bwX>Dy)nw+vY5^CpkVsb@dWNRTTC8wiIv9@X+LpGbS1SwdU zCWIqFq*PrB6jY$zk*4&pj1rb0&X`ntjiYNSrzoQl>i2p*05_o#8uWU-;4hD^`oS@a zN_hQ65Bb&>;$&Q8n@io6SX@Hq$8?f$(otP;SXKw2l~&q!;%xmq*pMaC5U^LD#^MLgvUCVNFw=SD2IjseRn+$e$1%G;=JbljWI~>c5XV?cGG=Ji!o<)^O=pmglMe6L?r?y9+ zw-1g5Lo`uGkRU=G+)1ea>KbWzXUYBgF3 zi^wUkP>g%4;th)kI%Ba~OlrZjWd65DqM_K~7>?|D3!6U*Oa-jg>nRPVP+CCGk69#; z;#EaBB~ccGULa7E$2&7xTaGH^OURNjKqY9IWj}O$M74%$CZYlkD#g>7W!b7by#T}s z852&D5ZF6~t!V`5-5;RmIGkGU(d3Lhil1r%X2V~HU8xVJeG5)w-? z&SF7ed*+ZdV`ulfM!nc^6i!KaiAH3sXrz+-=L*4uTs!EcfkJ2;v+E=Hc%&*DxRu`j zJXPmEhvk1sQ$&+19McHBh}kuCz;Igd+l{hfE-q>MQWrkS=|vH^WnIyPBx*Ski)Y}8 zHymG=dMl*jU^GeKEK{=0lreRJVNCFI% z`&=F&2S(+0bUhDCu|##{j;p*%H{_A9n#Hbb_*}8HqU+c~vyhyjIAAk?Io`B6U$i2M+< z5su$xGR>qDm&b7?rs(;}$=ee|uHcZCOwnS_8P{dY6i~xKV;W1sfp-Rir=}gSv{W#e z8x+;@fx>7;6q7*3OyM~JI4?99eTdmZL!Gg?01jNI`O^s2+W|$^!m`6m*@Yhq4SbdKom+wwq0S<<_gu4+9;TZrJqPF=R#4N;dkWYid z0~VDjg|5|ZE3Tvb=#A+mDV!JH`JuaQR->4*ERGb`7^R#^5~`yi*NBWc49z5qF$a!N z1n8vLOis;JBbIzz(>MmskS3bp%wb}Iu1OrLaaN)NXYrf-+R0mfhjDeQu(2C}ytja) zT7|BS0KLwn=90+=Re@d+&66ILT2x?^bZns?m7%wcykat3=U;F(JB(w0`4x{`#dIA` z6tO)jQIEvtQx9WavI4h zb*z>J{f?C-tMpX$LF+J)cbk_VuET!tw_v|o3z?pe-SM86k#Vt|6607HIku7q}XcebTfy(BjfoobTq}ugqR*{LBxs6k|{w}tHD6BYhd5=yaEG{wpFGcpV%IdPM!whL(n*r87DjH zXNU!~W0)C`WhYP&Cx}|C(O99ak6{3LO@*SiS!|Bv5itTsq{#|Kf-0T}Wod*Z$+e2B z4NZ9wc%&On4Q(fopBa!*IPVI?y|#|Y%4?kxB^AXnN?Bys4u&rA_!4V%fZvo|xOTJQ zI9|ZO0XI-#-2@z?Imy#$;xQ{qm>Z(^Z9t;^i;srI-c+@Cm zDZx=pgb3hgAg;ooa10Lzf9w5iP=KKCf6C-fE5m;kK;ZKX!=jx_meO#D;2%Bu&lL}L zhG|T=*U%7mBv;7a_0yhp)wu;@NQ6AMEb!A3XW?Z@r-^(VfjZ z6wf;9@77kUwY5C8w$O>Tkg8bBpr5o=*VamKT3bexMG+Zgpyba4@pKZJ5E#uYD1;FC zK*AOjcjDWYyEXE3?Z(Cm_KiLOM9UDw5)L%QT3Q3Ommz8% z{Z`X&zs76=rx{WLZ_9jr#U{sa!nM0ROW=t6m1R7^al6&h;+QdrmvMn~8U^u_FdKGfE7#VezyBQ_AT1@o&*evNQd3woe) z9Q&%iC7{iwFP!xT-fqwf_KXG>G>zvaz)TEL+st6Qv8g)HgJ1wQoB3s=CM`F|w*!E@ zOTjfq*IMo8quLMp2c5m(+s-(P<5tTvJTnJo&ad!G(x}NOjUC*?pM$N}5VdJCX6=?e zfM(~0bq3yNh}!$Fzi(^sgGTs5a+g&Fe@fb9KZ`i3ZKUu8}BEx89(9b`RFq3ei zxrC8CN*L;j<62@7i zeqE6y@~rNTPJ{)FXoA{JTd5ti3|&{XBs7en%c9a;epUu8Z_C$R$ZJMArdK2(LL758 zvfO)^o+_&ER*aTt277}&Eqj45fa7N*#`9AWvLq5iw5#({!YPaYEMLoo~YsQ!(; z{#~$if-lxe9Tw~>O&C7@-YP6-nxq*>ryA4mHJLa93Hww@*me_`oY(k6@%w8sJO4f# zpT9Dc#eB_TNfrBY{vyuA)O|lyLi#Huq`v|oU4LKHmDYxvsBWD%Y@pS*&tjl<>mYND zpZkyv%~0_PIe*0_-r$)p=l>-0ny+V^&Ptz;F$a@XXG8z|gX9>^a6-pQK+W$iz(_Gc zm~%YWz6h^vA+s#Dl-%R{%Jq#fdr6aMXf9q~@)W&$^XmEODf*wc@6bQry!{#d{Nv#n{5m{& zj-I`J`Q`_;txq)p(RWTlF_6H*YIAR}Z5wfdFYPIvbDA85SV<{aLO!JW=6*z`_=+;c z#wz9+(L|EO$c5H4L9?^1EtC+Q;VIIxB-G@G$~l;ok`E~}J_)7vZ4*;EnflhKBm!<( z8_Fp*^8%-7hZ7vlNHB{oT3epKhToj31t=N2k{OHWc&_Q48;?MZvp+t-_m2pgsl{b( z<1|H#bBd_%;Gh?|a1o%xWNt?tSbim5(a2Z|m<*7zsrC~1I0cW^@Yv_{sKiJ>jEzbX7(A$5A|8Otq-OXKRN@jMPu+%)jo0cR5QQ-o~r}a zfpUf?uDY54ItDHy(8}5%SBvIF;OMjk9JHVy&gjyGPeZ6I+5g~K4C9iRfjP)QlO$TN z9W6H1wC3cg_1Y}_pku|-Sq!uy*2aoRMH*N+u~fpS*jzTivz}MOhK826VrJO$Nz->- z+GEjEMk;~hdn`zF2!;{Hz_L_q1srPok@v*aR7sC+J$ zk9I!aPM!=ZqU0*2t$4YxY(eB)T(T&JE-YITO?6jG;&Ex{SL?Q!i5k&b(XF*WeL|9* zYNrwdwJNR`%*MXRjgScX^$jECAM z^jpRx*&M~%rCXzHS%P92%!(-19|hYXOF}Hswr$Z<>g?oSb3FX~9Sy9g(z?NUw}8{=wS&RSZp4%_~z7_rtu_AQe%4nQMN;^Z_J_wi%*5jtZj zGOo&?v;{?nmC@mr@51tA7hr*WVLEhy9(4^iUrCmt;ZroGTu2mH%({<|Ok+f~sk38_ zoXLT?q|Z&A!FG5?5`CfW!%t&?&)EzHXxs9D+iH|PpaAuE0E->;$=&B|M>LdZ$HV3a zEznqu#1wN9IfYJ+hal3ZI6&=ju}Bi~hrtwEg6d!uR824z%!yOIATKnJ*QhBudZh-n#-&r6+EZ8#JI89Zv!D@n1 z71z%#>eZ$dRGp- z#e!i#!}%Qr_72YSP~M%~$6O^)V*Uyi(~Y_ZeZhWiy(PG@Fw&|5kLlAEBMu|a_}Vm= zt0Wq#NbERieeH2#j7DDG1)?RpmUR#=SM?BYoS{Zt#YWnS%3S(JYaYU7rA-6J57QOX z5Igo>gg7<cWvrV^Stjp`60hw2@8=b+oC$A}Xyr4RU#LuFQ2etBQ>Dedq7>z7EfSx6 zKC^<>=jM?Av(p0kZ%X2naG_@9W^F8%|N2k*dwZq(|AT|Q{RjE)K9Z)4bT$NooH31w zoTudJObtsoB|}S6x{q4VKcqGUf%O)30c?YoBR!1-L{-YEhvS9xixWx~%lRTNNGpp6^aH6)2aTdpO^jn5wc#QzzG=(k%t)Gd84&|k4 zLv=@>5z$6vg3X8@L|(_~)yKCY;b4eu9ge6#+hAKo(6cwEsw^CNDG7Ii*2^(6q4qTp zV_+@(0Khaig1*R>H$IZfmkwZ9gfuSL5AO}D$wf@}JF;f^uuV3qkaMH{wG0r^|nJjXWL zip5%MNb*qCc7^7?NHjs2Fc#Tpl+jqyq(;l~xWzcQ5t)E4(PM;NC}FT4yyjF&l6cJm z&$23Ufhx5tzY@IK6k1xKuzDIaM~!QsK#ja@4(yv&qp2-!qh04i3~H@adJVmg zd-`vqy+fZs&!9Tc{%aex&)b`jSsig$>)=*E)++0?A~7wQXKSnDh1x#73qZ#JTIa@^mDO!;1fW2K+lvWlp-09yMf;^jZEsQ%CeijKO`?|qwN0GR zCgcp1otss+dNCi&N)=RjP=M8F7#jbq8;0D!FdssHvv%9LmPz{`HXvB%vglIyc6|Tq1yl6{>R_J z$NjtaFModi;+LK6fBaq284(iC((nEiecbyU{VV!7_6=P1%KTcD)=-Z`q3XK@{1}nBSPQ_6amc6=!~5Ilgx9>92R+g zp=)2yzK+n>XRT!elRtR5_Sh&Cp0*1k68v?ijbOl@x{(Y1x`Sq70wrf+0%gA1v{x_A zzW#Ce_3Po+r^`C#{AjDaGJzj&h(Pf5s|D22@JDL@Sr>L4$YTx)xjc@0b)lQh`l963 zTOgCQ*6?LN;J~@|u2mRDWaSEHoX%D+t^J9iCj~*HmZWHR9P~Yv2n;?@?$&}<&@nc8 z#MB%&I*^c|tT!}uap6Ep79?c?e)DGP8d!NS&+ugJQ3Q{mWy7~*P>gL%MQPR4N5A~a zom;nK?dqhg>0&k9%(JRWK{UM)g%C(4Zapa41<25k5eX*2uw{p>5T71BeTv%c$A}Y| z@kF3@dqKHUBzvz`P!4jxf~L<0e<-xPU%OLCK0`&QReun6Ee>GugyR_98`w?q>VMs+pL2w*rKO*#88FQQpN zXq~prEX*o3R@yuxgCP(tK2>6^;BI~XMMd)p!mWhQZTD~JkwdBRvFkAW!l)E3o8{*eh8#ZfLS`|EQqxJlO zm@jg0JoIj7Y?iiYgr{+=L#w+FOh4cSpYqJ2`?&ut(tM@x8_(G{uec9bzss6T^1Amj zBWn#xJ`d%w@pCFgOCwdyvIFMl*m3i}<-_NHtwEtQnD?}POgS|s_dv2Y!s>oULUhM8 z!7=@Jg~Z&n0XMWzpn5_~k~8Oe6i}EG8qGMCK^@!qF>y zHzH-Zgw6UAyK?Jzc{eq;p#NJ*|7!+&Q#J6i_`gq{9F*d}^!xqZgZ}qE(w*vmzt`*L zrfT4u27}2UM!MO68p+~gW;(ROCWw`z*(`QmF%C*vx6NX5LdFUyOJ^1gZ_5Sq_q6L9 z;amtNw{ZXSNZjz^$E0QZe=r!7{eSy|hxpI;lFG&-a6BEsjkA5^-`(eOcu?!B6wB#j z^wnrCb&P}np6LyKh}j6W)vD<}f^}njbbf|L1Dxn^J*g z@!$IUW&i&ty9W>VzxR^-!yTuo$j|X-ZvN}f@8+Yfa!PJ`M&42sxZ03f=Hn4OcWR=A zo486kXpmyIU%97oRJy6tY7>;jE~v;I?n4{R$V~<+Jl}JhSZ;zRs*A~#vdBD7>%m41 zW<=i>0WFI4!X4zEYq@|oPh^1y= zPF9{PcMVn^Cts_(UYp)ie}$L)-Jk7`^^Yn$91r0&56+2swGFi*JeD?Z- zBD9ooS%PGzY$|11$na+PnDJulbyYY}zUu+Cemmw%6ZobbT&7Z$ppQ9LD~xAN^~ESO z5&mkhc2o{vxH8RJ;W!w>%k18!vtKr*jFmS0ppkjt(&}5G8?{tc!W7>jFH+xXUXis~ zT))l%v#j@7*v=Z9U!kj?o7a6v)zmEi6(4}Si7IHB{-@vXmE(W!_685~-+iQyAM1D| z_rhqds)2}<+m@*9ps$Q}=V5REp;Pnzw_p65jK)~;pNIVI#1mLU0r$q8!L}ZD8SPFzh8d;qyJ?8;9>vYN7`in>!q-O z0=(60<)$=1gIP=kG%kNYI$#k0+>~JX{`V}4H!1KKA`t9#F{KA)hZe{o9Y9LVX>W|)o zJ`K8_J+$aiz|?=+R}tWX?gf%rgqd@qN3m284ktve@u2-)b5HQ6dGcJ#pW zy|xe_qDI{-qLI61{4!=ZdQBv@(SOar)n?sb)-Ak-W^SVir|K`YQT@JVBP$Gi)35_2 z>F6Uwm;BsR?;+W={QS=Y25qVVTyp;R`%fzRzk@;l;rzdk<3WVNFL1}m`cjccEr6P3!U}8bMs!hSIKli$SNIEU2 z|GD$z9ohf)_jW7tU+=;G|9(>ClG_q6HSL>Ae+_D%)g{0mUGpmnfL_!x8WUaB@V?%M z8*tXzvMDM+E`D1ZxE`<*qOGofwqii?ayHAP%6dUb934$@GHDQK*MZO^jr?Z_J~tH3 z|E#Cfa{7Pc=f9WH|J^5*`@i1c;DP?%OIoh~zqL2NH)m^aX8zrH#Fa&ix>$~l^rj*y zuo2T?MZ|==)t9^O z_bqLPwb|#jS@yO18|H_En|1qPjUKPT!7kFI*ZZYo8n!Mmr`M>w1#QzNrdV3H2 z-+iQyD~0mcvyLylw=%wp+2k0@DH=ZY9+$%Z&NzW5reY#HIwFjIp~uzBV^9C&o>^b| zcn-ewUM>Jw^;&_FQ(6aK3daXVn(ZjRLN`S4U07a^ z^4gP{WlxE;7_H*sY@=;>9;xnm`I=>1R^or|ZTQ*JiuC_RN`Phde@_m|^uIrNi2rau zX(9c0tYCrgZ%V*7B-^VJr?n_vE(WgAf_s=_5%|4(A%5n=BC6+4Uggfx&HoDfpWHeB zH|Rg`f1fF>!~c=x{GWMW@5bbUY{>7C4G70(JkyuXPm+cFq|V?%q1)Cfto9~Dc1MgL zxfe!|EToKojo;(V zp$8B1h&67Es;>9eJi8qGXF|XY_sz-uHpRFH0-X z|H&n}YyEH8|8KB=@F4%+OInBjpInk<{6D;q4sS}`hSqm-Nj8!KCYR(v3IHiUP1zzT z;7ein)lA<#!>>l;;W#Ly73u%o>VKb<^*{YV{~`Xvy`+`+|03bvl$75w0spf2|2rpw zYjDAf$leP3|8iG!-Sd`r5dUF!(BCWX|GoVO`Tt&0jsI8U`n=rDrL=AG+iDe@@s|9tDx{0RYAZLe{2w)#yCUt2?)gd)8{8Yy*zsU zixYrf#F?1-PSdFG@_yhrB!#?vm-h@_oAmSEj>j(d6`P#DB3C4KxilFwmwm)IJ3o~; zygWap|5i;5_V%9~xO}bB73VK#Y}!xt7wC)*)%gD9o8xz9c}HLn3&~87D{+P=#R5nV zxNTDw$H6yDGoW6TxmScvb_`MWnS&z`EaW0W9Gp+gt0J0Uqu z-z5>@B4oVi3V*8d&i%}{$)Z4^E8yDE9HzxskMxU|ug;#IEE;M(J+8oOEW_epK5}{g zB=hqkfY;AYPY-{1UYh;FJjHC{vQLP@j+{Tk66-H-lP?8Pa}50X$Z)~=(3oH5xktW8iZV68p_aTVtMW4r`+us7K2c^#B|e*TKis4eya z$MC$SPg)e6Tu66K;VR*Mki*p z@RkKE-&0cOYu|3*C_BHhj3+pbI(RbSWTMDeyBJ`sQq@w}wv%3$J}R-0tF6Of>4Olp zS3=>GgqI?l8A-0-i?y9{nn?6jaAx2J-(x|5VQViE1KT@5&BYWC_76lh%UPz)Mi!w} zM^~3EzkIRDP!^*pEqV&Utu-%HQtcB6&05HpsU9*i@cXxT@&K^UQA5?a+maTzWqXzdEz>av(Jp(nin zH@&4L*;%w$7ByX7f#tvr?g%Ztn0R03Xg%=?v6L|hQjy;9u}XMH@uq!&mx%T2l7%N@ z3vUJI0r;SUHh=xqT#Dld&kkF94Xsh4l{4SYIgkD=&Oor$r zOAf_hqF=byXUqv>5?*lx^5jk&u1dZWgf~<-tU^<6mZmY8kwoIy-!+wj9{KL%6dHb! zSt{qxC^rv*huplRY-!$~Hn7KvX+`<}PVIm8_bUFM`@6dj^8bCLb?ko>RbnYgc(~LI zX!lmUKC2@B>i0np_CJ#$YCGn1ffsL9C$t>KRyR$Cxw#OgG z$Y@3SZ~LD2ag8%z8U63~%k+P@{}BK6e$qDR-ymdh;(okQMsWQu!Q~(27_|^zrEi6gZ<~dq_y~ew-9c83}hp6r~qxo z-a9=0P+Wd~L*%-znD1jw#`Hts@$~}t-}W}3+NMt1ypjaqsg~E74MpqE@WdAqhNz54 zx4u&^j2z*0$+{ZWlYrFlcsZhIY)u%t7FeaZ- z4l0xR4SfW7C4k5+`Uvn#^0{0CicK%|2w1BV=+3P0D_H*tjPF;qY;QW+YE8mb`+T8= zyT3OZEZ6fA3(zv$KrI0KN_8oFp~ZfQPg~W1A0d&AG{j~IU$~z`qYZj_HW!*}ue44y zZjOk)NR&BO+;aEdX^Da*X(VLp-a~lb14X=WF;t0o)WD>{c zzH`O!DZ0^ZM%B5Zrjb982gfaY5^Q?fenAcd^2 zErwFcaa&`h-bVDN6$ibblwW(q>(0hO7e=59Hdf{>HBZBR#{rQ1kWqi&h*EzTL?1$5 zEyMInyM*~q3VpRI|96uMpk?7M{a6D;&X*= zs*&I}irutmaM4{}8#RsQpWp)N3%}Jsf8yv(?Pbh+yS7oo(Qnx3L&&IG3K_MGnB8T} zDAo3t6Ev#2AQ!c}#;X5Pqf*_RN%*pEl^Qy87xzmGw#*;krm1P5?(oj3p~+jheQJPx z=l4*z#9==Kr~3TCsaBEy8QawJTc>6I-@AkTa{P~8|H1$3UeY@L-)bRv4q#5)dXnY4PD_9WTk*k^;J<_WT)n$6>h!2?3%u9JK2+y-!jUH zzFNsW)!QgA#zo^&ju-MIAEVqlsa!mGP56YIzhcfU+Uu^xzatA>H!02^?EC}X5U zTCIzf0d;GbopCy=qNcuXfSUn!E65quegQT29{4(lIY2dNEqWC8_Pxqs=WBPt&rMYS zKxx(U|0WLrE;;}A_YSJ>e?7#1yq~m^|95`y-{cg|PpWI0MC@8T*D`W`KqV|9hiWss zGgY^7O%rVtU=GHVvoVczh63d2j-a|zwdT5o$X+D)>IPqVtW!}vzo;Z?P&ntQV-?H= z{jwHP&J90w&dG!d$>&``8*tF>g%_4Aw*a{?CBnr#`bR!8Q+T*}E;kav6yxr___ zGAfON<|&`CK*o8%vRnfNl3c?nwW_mG?NaFEu96!phk8~>C-wFG3{TFj1!D1nz%)`0 z1*9i1xWYV$=_oLC6}l3$NsueWN>O@rG79CohRQL5nye`+cM^5%fNMPXUb|f$O|!(< zBo&rNCA%=~s{#CkBs-OyCX?!wM&%M=b(CM@56uYM%vI5Kz-}AaqI+zib|GsiqD4Tr zj%F518qx=7jw5(p5wm(;o5j=MIo1TTcn%pLf7Iis!VNbMlcx~_ zU_N(SV+U9EB3Qw6dEFkEFvTQ^QbrSNuaXuIFk`eeBa+ilv}#3w=1%KsCj?rj$;9=# zk}>p*{+dHH*UAPf;=-#~@Ri%&t8L${rqk-I6ilwxT9dbBF0z#VV=b)RL=0F$|9kuU zW&4lbVDJ$C|6Wo_M*v1|3v3rxH@sY8RR5*Y=Azys@YEZ+wSvB*R!dPP%3p$)S95CK z|B?uK&Phn<6}bubx7`1KaBxuG{}1nf+*8v0pR@cYLY~+uw`8E~p>JpyZK1LT)3fH0pFu6GnN948Y}4?-0$TVixP z{bIxYMrnw&$^HT&=E@WSlzcSm2DlEKtL8f@4vL^X5?dbf%33VV((y_J`O9Xc{-CLu z|H}m7oW-|r0klm2H`p!7fBh%Dy$AdMdr51`e=B%q8Mdpzg>4m6u?IlYe6x#w?4pUs!>>cF;K1)@U8XGMQ#l z{q3CJIw-W^Q9CFiF)92RvRLss6y8hgpo#Guim8w`LS8t9zp^y4P_E(Q#^{QC&PSpxN$pH9K)s6P{hUbcs>suL}H7cq$Z|gw3^VxvoMUQ2%komY^<|ZzX zfje&W4}H3P?)~Ds8R{@d9>J@gG3D`0d-Zs{-H-`Bm6YGF%gkuKdMa-^m1<+d{Z#M&V17sjf zMAv!E9BN0*8-t2;c;FjBvJZN)Op=?8&ns*Cdjo9VSgY-up>m_?)|i}vp!Vj6o3F%L zc(;k+%@=_Mb!)wB>Yh>KHrYjaCFvMVvG^yM+ordj6G^aR%KfE;U7?8LuRM6xe7)|M z#_Fr{$LK5afeHy>7Y)n;M%&~!R|L9Nz{G?Lh*0~){&jTnOd!k&no%LtxS?aX6F`Oq zKype@$T%k=We|HuRYBntbBRV|%(yG6^5m2Xof*Ixv9x>MkI#KB{#nxM{NLtkfMxbS zdwUi8pS`^Y`_FqxTWf_#v85SD+fyt~v+0Spi80`@iX#*;BA~G;z9PWSRKTFaloF0& znvkFsJUcx{7afBRuN>8k(PpW-U% z=72^xR1|MaVY%y#ELL z`+I}Z{@*?5J^26LOWHzZBaBw-zu#*ETAZda4Pm?fvOSeD6~k_q&L)AFQW8hN42*Cz zAzf9z;}!4FWX$*sYhR{+2sw|*({YStNBovyPIl0LzrV~z1YOz7weOzxgTWx^b$Uyh zQq{g5^i0K$tJmo*Z}k0tzsFqCF%Bh~W}_u7*`i+$dedxVcSy$#)YA5H_!iVO8!7mf zqIT6`3q5lCuRI$&t(MVLBKrxA8Ao~eAvD5*M2IEGs9C|A&uMRiR_oEDAK8qgctVDc z9-+Tmr2KXj_0-6H*J^ETp|e;2>)FZwYPBvdE=E{PTMAL91?qGrBvH7KPJ(9ybvoCW zO2k4s>~<&9kgGlxdZns&fLRii9;!B4Eym&eF)hH6fWeHEj%nzY$-N+iIvvv<`s%0W zC#NspzR@^tZK0QmCw@3E=^B?wAvht_77uv|1g1 zk@bWA-~MWbz$y5SPM(QY>zyFzLK_1eWkRyqiQ%1}g3|Fq&!$2IUG1t_##f9+Nb)&g z$&4Y9);^9on*rt*A&Th5NbvPh|G(&8pifAF(U!S> zNSRMa*dRg|?bjnX9ExzT8 zv1p7Qu@GVgha{DS;K4c?Pj?Fp#ngD|4?BRk{fKbP36ACn)ZOZXE~U3l!Xl#4C&?B1 z1a<)m-}k?``7=?x7iH?uPW2@q0@NX&i1)o;ibK-T!fq#q$5cARn8iH>gNowv@vY)( zTT#UAt(~{(TbIJL7#q3GB%6(h zU(dK67Q5E=`qLhM{QYs;MI~tu+izGx+K=1DyAiS#E7l&io%2&02FJK+=}@2GVh)4z zTSoXCTDJk8G(H70;Cw{XnpM#ByZ%th)}JA^Eru7s3q+DtP(^j>MR0e zbP49v0qsM=5(I(3GC{cJ8ZsK8ey`U9XcHQtL9f>f{_^Om9~@VP>o5BLb_;Pb*3k(X z`Yf@y6y`y+jFXP)h{Lfu0x&yb>C_y#Uf%Zv!Rq0ZL7Ooa0=S4OhHej43J0c-jn2=T z)7zq_xS=ne0f`NE@;^~!7?0m|Uz3^QcWg1Z^dUm=>U2ic>8Q>X|NW?j?d-o~Ro-)N zSVYj7UqBr(W1x@o8h}v#(BSNPRTL$%Hr*4c*Uw zbIwfbCuFQ!BA&4qv@}+u0Q>c5O2W%$S{a`M$@49jp#eEhMGMCKtN{SBvtKN#6w)^t zI^M4tscCLP235^0x}M~L9d*9Ar0L59?$+FNI{3oaqv)d7E<%VOJ)|xVaDioKsZ74X zES1w&IQ~jobOKl5s6}h;K!5?V;(`w_?|co9cS6R5lOzNtQtbu1;B_d^06oXy)bgx3 zB!NPl7&#T;-1QW>7ir;ausIPD5=%19VnJlFyA5lomg6&*J%z#34{_St9-O8z2(Ssc zc5N>h?7E4sYx?J@Vt7CsqN*UO>mar3pr@XRmrqA)JGy}7c3D<+VzCeO!Au3hh~w+h z7#J|Xn{$?pVj`xDfy9+&JY~W%emOFwEmzm+7cmA+fW2hPXE|iEG$xY7^LnT?r1yo3 zz*@vzpv9UmciiSxx*?B*yKr=ko+~C>bZan**6(GsxQ|g|52HKjUR2jPOPFeK(Kd9Q zH$V6rue4}V$WGl6g%Ls2BBSfYqV$^ZL5 z|5xgJD1?(aVz9jwh!Qm{oI^D`<_Dnux`dO4#}MfH`N_%K69lIgaI!w!!l6;&LNrhl zj~?m*VL>d>y|C0KnVT*!ZVICr38y%rVpi-&>1e35sN;wyc>&3^Z$wUFOdms#jLY(T zeLD*{vD<2}z}pn(r^My}8|g0mp}3u|5z!(%TyYv@qJr}7IgfMS%BNfu)cvXcnV zNwJxnUV#%ACOi5Hr-s{TAnFmh(&YSlilyizKIs6=5ICLy>3RI zuTx()rbQqp7)pk?sjjt%4XjD0q)+KlZ4{|$FOFawgF1!^`7 zb1KQ(BqVJ>SxCkeQeljRR{sDnq}K*Cx2Pq1uTmCJ^2c1;x6q2MUTC8K11-Sb6soUo zt1OrsAbxEK>T4pg71cyt8w~}QfL#gXa&S&*x)5GnDGy+-LRmzga$mis%2h(8GCeYt zn7h^Gpl+LPRY2c_Y*m2W9@VOVz5}A=wQ<8mzS2}~xZ+oWJ*F|ye^v*^-7X-%R1qxs zfEDX7^g?9`_|j2zbW|-JTG>A1U(U7~IrC&Td>iSPt!DVcxqz3jypJKg&Xs=_moz<( zSu#0~^D^Lk)sd$P|3#Jd%1y9ErT0svg^XC%G?2N1Lf=OP zVVafLi(YTGYUP=}>?+jX@4aps@4UXf6{|KsSfO=SyRQAgzTZArx4gy)9jgkJ4g6eX zE2+rj=W+-ZO1Q$DVn<1rd#vPC7_Ka8vl3wIyp7F*k?m2Oimi~*Ho;L{s? zS?1oARJQ-dY1&cre@1T427Iyocfa4;t@wW&>_7N_-Amd+Cwc|gE5CwOD|Obz_&i({00LM1B`R6NDX!Vccjrk)X*WCZ4LZ)q24=npwLLV_dFi)RbK-$eA#@GJ<8s z$z#OHSYH;hjHCBzfb-$EZ_v7-w@*G^4T29{lGrIwzn^q>v;pnAbi#3DEOR{r;H!Qx z*#B#*bt*B}29`-B`!Ra&+b1F5Qz#Lu*|%83mTtkuw4i2ySG=#jZZ$rg3((IYr0z+QCBiI}=W1}3jE5_URw9-B3%mI(Qf zYUR9SC?#CYAZQnpj<(3u7#Pb;x65D|UNpK%UO>H-e_pH8TL>0XxEGi zUY!=2pfSEuV>!;(_{83GkvJMI>TDJ3F>EV4baS=P?N+K=qV~!R&{2~O9-E4*z~}

Qyr0?2;r}OAP}? zi`<3C!l@qIamGPm4KOrOgF9SkB4H9lQnYOlPRv@ocwFkjV?<;a>;!70+*olv!val+ zL_%iarMut(%pRju;~cU?Z4p4x9s_DmdIZM}2EZ-!SZIkqXtmy@EOAWHqJ=D(mdhS{ zhROO83f4}|^3o@O_HUt5t@tU9$NAQ`H-FA(!8c{MC}~S#7mUQw2iTP0VWCQ>byB8a zhY^HA@g1PnCt=tE>F#)Ud6^)Q^Db_jiLpHoA|f%33sd7xxO~ptpJ;MElqdQ3) zK{Y}DkaxdoK54AXZOCdkC5R-Fa<_m=DA@*b)asiA2bd0eZ?76(#O!)g{-3giBlV@oA!n;hm*fF*bV1g*nwfdn=BJ!bd8>l!8hWYXvfmj+NlJ7)A z)4=hjaAE8lbvZLve{}Rk)oqTCo{icO(pfcQq_HTEQogCwic^0&eo^1va)AUGuFXLr zRAPg)!>Ni{`xH{f;D!UJL9*nEBoxf&*w{FD;hMfep9qDs1s5lhnSTL$l0!uVd*0pN)*P> zn<(hOEiT*YdQEB;>Wx+?^wsDr1l%@hb4y!jxsZ_`ERM^8!^M*}T!!%%2Woh#6|F^+ ziPkBb6*%;W3hg5R{7%eZef`QZp5QnNkm6Dqk90G3zVx|Z%(@7=Fe=?wY;ug{^a56@ zFy=jaxS|04Jf*Sj!bO*7AnkNbI~OQ&v4Uvs)Z;WiLgD<1CfSD)x)B5B%rY1-5>$Ye zK0s<%jg}g{8)b>i3hoUTPK$4`2oO3`drJ~NQ`-y9@k-}-mY|CeOVlMY?3&!J8!qC&IhVk~J)ir^N5Do;}YkZ>kh5sJ3NrQ@2`wFjVpcoRla0@b0M zk_5!(YL&;HrH3w2!K2lBIR@@jEBEvo5Fw`{5+Q3-V$pkC)~V!N{fHpAi1*gEF)l-C zj1yu{5r%LSaUz6XUsQ;UfOL0Z=Ju&Kya_^u)u8t2h%u?>Zid4tO^6;408yR+Sew-@ ze{R*(L%N4l*8jr#7M(nZE|9aEJpr&J{!hPOj{ntvvcGrmp#Qy({>g*`7gE$qu@1b-&h{5Fh0dcv`C^^%0kGN?ymw~U$IGB02CkS(v|cD xU@?Qv$R(K%QD4-3KflIQLgwdQzYpmlJ*0>9kTy&Ie*gdg|Nk^%y`TX60RZ8^KfwS1 literal 0 HcmV?d00001 diff --git a/charts/rancher-logging/104.2.0+up4.8.0/.helmignore b/charts/rancher-logging/104.2.0+up4.8.0/.helmignore new file mode 100644 index 0000000000..b3123b9ea0 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci/ +README.md.gotmpl diff --git a/charts/rancher-logging/104.2.0+up4.8.0/Chart.yaml b/charts/rancher-logging/104.2.0+up4.8.0/Chart.yaml new file mode 100644 index 0000000000..e74538294d --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/Chart.yaml @@ -0,0 +1,30 @@ +annotations: + catalog.cattle.io/auto-install: rancher-logging-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Logging + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: logging.banzaicloud.io.clusterflow/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-logging + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: logging + catalog.cattle.io/upstream-version: 4.8.0 +apiVersion: v2 +appVersion: 4.8.0 +description: Logging operator for Kubernetes based on Fluentd and Fluentbit. +home: https://kube-logging.github.io +icon: file://assets/logos/rancher-logging.svg +keywords: +- logging +- fluentd +- fluentbit +kubeVersion: '>=1.26.0-0' +name: rancher-logging +sources: +- https://github.com/kube-logging/logging-operator +- https://github.com/kube-logging/helm-charts/tree/main/charts/logging-operator +type: application +version: 104.2.0+up4.8.0 diff --git a/charts/rancher-logging/104.2.0+up4.8.0/README.md b/charts/rancher-logging/104.2.0+up4.8.0/README.md new file mode 100644 index 0000000000..7f66f42ab6 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/README.md @@ -0,0 +1,134 @@ +# logging-operator + +![type: application](https://img.shields.io/badge/type-application-informational?style=flat-square) ![kube version: >=1.22.0-0](https://img.shields.io/badge/kube%20version->=1.22.0--0-informational?style=flat-square) [![artifact hub](https://img.shields.io/badge/artifact%20hub-logging--operator-informational?style=flat-square)](https://artifacthub.io/packages/helm/kube-logging/logging-operator) + +Logging operator for Kubernetes based on Fluentd and Fluentbit. + +**Homepage:** + +## TL;DR; + +```bash +helm install --generate-name --wait oci://ghcr.io/kube-logging/helm-charts/logging-operator +``` + +or to install with a specific version: + +```bash +helm install --generate-name --wait oci://ghcr.io/kube-logging/helm-charts/logging-operator --version $VERSION +``` + +## Introduction + +This chart bootstraps a [Logging Operator](https://github.com/kube-logging/logging-operator) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.19+ + +## Installing CRDs + +Use `createCustomResource=false` with Helm v3 to avoid trying to create CRDs from the `crds` folder and from templates at the same time. + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | | +| image.repository | string | `"ghcr.io/kube-logging/logging-operator"` | Name of the image repository to pull the container image from. | +| image.tag | string | `""` | Image tag override for the default value (chart appVersion). | +| image.pullPolicy | string | `"IfNotPresent"` | [Image pull policy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) for updating already existing images on a node. | +| env | list | `[]` | | +| volumes | list | `[]` | | +| volumeMounts | list | `[]` | | +| extraArgs[0] | string | `"-enable-leader-election=true"` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | A name in place of the chart name for `app:` labels. | +| fullnameOverride | string | `""` | A name to substitute for the full names of resources. | +| namespaceOverride | string | `""` | A namespace override for the app. | +| annotations | object | `{}` | Define annotations for logging-operator pods. | +| createCustomResource | bool | `false` | Deploy CRDs used by Logging Operator. | +| http.port | int | `8080` | HTTP listen port number. | +| http.service | object | `{"annotations":{},"clusterIP":"None","labels":{},"type":"ClusterIP"}` | Service definition for query http service. | +| rbac.enabled | bool | `true` | Create rbac service account and roles. | +| monitoring.serviceMonitor.enabled | bool | `false` | Create a Prometheus Operator ServiceMonitor object. | +| monitoring.serviceMonitor.additionalLabels | object | `{}` | | +| monitoring.serviceMonitor.metricRelabelings | list | `[]` | | +| monitoring.serviceMonitor.relabelings | list | `[]` | | +| podSecurityContext | object | `{}` | Pod SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) # SecurityContext holds pod-level security attributes and common container settings. # This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | +| securityContext | object | `{}` | Container SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) | +| priorityClassName | object | `{}` | Operator priorityClassName. | +| serviceAccount.annotations | object | `{}` | Define annotations for logging-operator ServiceAccount. | +| resources | object | `{}` | CPU/Memory resource requests/limits | +| nodeSelector | object | `{}` | | +| tolerations | list | `[]` | Node Tolerations | +| affinity | object | `{}` | Node Affinity | +| podLabels | object | `{}` | Define which Nodes the Pods are scheduled on. | +| logging | object | `{"allowClusterResourcesFromAllNamespaces":false,"clusterDomain":"cluster.local.","clusterFlows":[],"clusterOutputs":[],"controlNamespace":"","defaultFlow":{},"enableRecreateWorkloadOnImmutableFieldChange":false,"enabled":false,"errorOutputRef":"","eventTailer":{},"flowConfigCheckDisabled":false,"flowConfigOverride":"","fluentbit":{},"fluentbitDisabled":false,"fluentd":{},"fluentdDisabled":false,"globalFilters":[],"hostTailer":{},"loggingRef":"","nodeAgents":{},"skipInvalidResources":false,"syslogNG":{},"watchNamespaceSelector":{},"watchNamespaces":[]}` | Logging resources configuration. | +| logging.enabled | bool | `false` | Logging resources are disabled by default | +| logging.loggingRef | string | `""` | Reference to the logging system. Each of the loggingRefs can manage a fluentbit daemonset and a fluentd statefulset. | +| logging.flowConfigCheckDisabled | bool | `false` | Disable configuration check before applying new fluentd configuration. | +| logging.skipInvalidResources | bool | `false` | Whether to skip invalid Flow and ClusterFlow resources | +| logging.flowConfigOverride | string | `""` | Override generated config. This is a raw configuration string for troubleshooting purposes. | +| logging.fluentbitDisabled | bool | `false` | Flag to disable fluentbit completely | +| logging.fluentbit | object | `{}` | Fluent-bit configurations https://kube-logging.github.io/docs/configuration/crds/v1beta1/fluentbit_types/ | +| logging.fluentdDisabled | bool | `false` | Flag to disable fluentd completely | +| logging.fluentd | object | `{}` | Fluentd configurations https://kube-logging.github.io/docs/configuration/crds/v1beta1/fluentd_types/ | +| logging.syslogNG | object | `{}` | Syslog-NG statefulset configuration | +| logging.defaultFlow | object | `{}` | Default flow for unmatched logs. This Flow configuration collects all logs that didn’t match any other Flow. | +| logging.errorOutputRef | string | `""` | GlobalOutput name to flush ERROR events to | +| logging.globalFilters | list | `[]` | Global filters to apply on logs before any match or filter mechanism. | +| logging.watchNamespaces | list | `[]` | Limit namespaces to watch Flow and Output custom resources. | +| logging.watchNamespaceSelector | object | `{}` | Limit namespaces to watch Flow and Output custom resources. | +| logging.clusterDomain | string | `"cluster.local."` | Cluster domain name to be used when templating URLs to services | +| logging.controlNamespace | string | `""` | Namespace for cluster wide configuration resources like ClusterFlow and ClusterOutput. This should be a protected namespace from regular users. Resources like fluentbit and fluentd will run in this namespace as well. | +| logging.allowClusterResourcesFromAllNamespaces | bool | `false` | Allow configuration of cluster resources from any namespace. Mutually exclusive with ControlNamespace restriction of Cluster resources | +| logging.nodeAgents | object | `{}` | NodeAgent Configuration | +| logging.configCheck | object | `{}` | configCheck provides possibility for timeout-based configuration checks https://kube-logging.dev/docs/whats-new/#timeout-based-configuration-checks | +| logging.enableRecreateWorkloadOnImmutableFieldChange | bool | `false` | EnableRecreateWorkloadOnImmutableFieldChange enables the operator to recreate the fluentbit daemonset and the fluentd statefulset (and possibly other resource in the future) in case there is a change in an immutable field that otherwise couldn’t be managed with a simple update. | +| logging.clusterFlows | list | `[]` | ClusterFlows to deploy | +| logging.clusterOutputs | list | `[]` | ClusterOutputs to deploy | +| logging.eventTailer.enabled | bool | `false` | | +| logging.eventTailer.name | string | `"event-tailer"` | | +| logging.eventTailer.image.repository | string | `nil` | repository of eventTailer image | +| logging.eventTailer.image.tag | string | `nil` | tag of eventTailer image | +| logging.eventTailer.image.pullPolicy | string | `nil` | pullPolicy of eventTailer image | +| logging.eventTailer.image.imagePullSecrets | list | `[]` | imagePullSecrets of eventTailer image | +| logging.eventTailer.pvc.enabled | bool | `true` | enable pvc for | +| logging.eventTailer.pvc.accessModes | list | `["ReadWriteOnce"]` | storage class for event tailer pvc | +| logging.eventTailer.pvc.volumeMode | string | `"Filesystem"` | storage class for event tailer pvc | +| logging.eventTailer.pvc.storage | string | `"1Gi"` | storage for event tailer pvc | +| logging.eventTailer.pvc.storageClassName | string | `nil` | storage class for event tailer pvc | +| logging.eventTailer.workloadMetaOverrides | string | `nil` | workloadMetaOverrides | +| logging.eventTailer.workloadOverrides | string | `nil` | workloadOverrides | +| logging.eventTailer.containerOverrides | string | `nil` | containerOverrides | +| logging.hostTailer.enabled | bool | `false` | HostTailer | +| logging.hostTailer.name | string | `"hosttailer"` | name of HostTailer | +| logging.hostTailer.image.repository | string | `nil` | repository of eventTailer image | +| logging.hostTailer.image.tag | string | `nil` | tag of eventTailer image | +| logging.hostTailer.image.pullPolicy | string | `nil` | pullPolicy of eventTailer image | +| logging.hostTailer.image.imagePullSecrets | list | `[]` | imagePullSecrets of eventTailer image | +| logging.hostTailer.workloadMetaOverrides | string | `nil` | workloadMetaOverrides of HostTailer | +| logging.hostTailer.workloadOverrides | string | `nil` | workloadOverrides of HostTailer | +| logging.hostTailer.fileTailers | list | `[]` | configure fileTailers of HostTailer example: - name: sample-file path: /var/log/sample-file disabled: false buffer_max_size: buffer_chunk_size: skip_long_lines: read_from_head: false containerOverrides: image: | +| logging.hostTailer.systemdTailers | list | `[]` | configure systemdTailers of HostTailer example: - name: system-sample disabled: false systemdFilter: kubelet.service maxEntries: 20 containerOverrides: image: | +| testReceiver.enabled | bool | `false` | | +| testReceiver.image | string | `"fluent/fluent-bit"` | | +| testReceiver.pullPolicy | string | `"IfNotPresent"` | | +| testReceiver.port | int | `8080` | | +| testReceiver.args[0] | string | `"-i"` | | +| testReceiver.args[1] | string | `"http"` | | +| testReceiver.args[2] | string | `"-p"` | | +| testReceiver.args[3] | string | `"port=8080"` | | +| testReceiver.args[4] | string | `"-o"` | | +| testReceiver.args[5] | string | `"stdout"` | | +| testReceiver.resources.limits.cpu | string | `"100m"` | | +| testReceiver.resources.limits.memory | string | `"50Mi"` | | +| testReceiver.resources.requests.cpu | string | `"20m"` | | +| testReceiver.resources.requests.memory | string | `"25Mi"` | | +| extraManifests | list | `[]` | Extra manifests to deploy as an array | + +## Installing Fluentd and Fluent-bit via logging + +The chart does **not** install `logging` resource to deploy Fluentd (or Syslog-ng) and Fluent-bit on the cluster by default, but +it can be enabled by setting the `logging.enabled` value to true. diff --git a/charts/rancher-logging/104.2.0+up4.8.0/app-readme.md b/charts/rancher-logging/104.2.0+up4.8.0/app-readme.md new file mode 100644 index 0000000000..994c597ee5 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/app-readme.md @@ -0,0 +1,45 @@ +# Rancher Logging + +This chart is based off of the upstream [Banzai Logging Operator](https://banzaicloud.com/docs/one-eye/logging-operator/) chart. The chart deploys a logging operator and CRDs, which allows users to configure complex logging pipelines with a few simple custom resources. There are two levels of logging, which allow you to collect all logs in a cluster or from a single namespace. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/logging/v2.7/). + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. + +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Namespace-level logging + +To collect logs from a single namespace, users create flows and these flows are connected to outputs or cluster outputs. + +## Cluster-level logging + +To collect logs from an entire cluster, users create cluster flows and cluster outputs. + +## CRDs + +- [Cluster Flow](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/clusterflow_types/) - A cluster flow is a CRD (`ClusterFlow`) that defines what logs to collect from the entire cluster. The cluster flow must be deployed in the same namespace as the logging operator. +- [Cluster Output](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/clusteroutput_types/) - A cluster output is a CRD (`ClusterOutput`) that defines how to connect to logging providers so they can start collecting logs. The cluster output must be deployed in the same namespace as the logging operator. The convenience of using a cluster output is that either a cluster flow or flow can send logs to those providers without needing to define specific outputs in each namespace for each flow. +- [Flow](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/flow_types/) - A flow is a CRD (`Flow`) that defines what logs to collect from the namespace that it is deployed in. +- [Output](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/output_types/) - An output is a CRD (`Output`) that defines how to connect to logging providers so logs can be sent to the provider. + +For more information on how to configure the Helm chart, refer to the Helm README. + +## Systemd Configuration +Some Kubernetes distributions log to journald. In order to collect these logs the `systemdLogPath` needs to be defined. While the `/run/log/journal` directory is used by default, some Linux distributions do not default to this path. For example Ubuntu defaults to `/var/log/journal`. To determine your `systemdLogPath` run `cat /etc/systemd/journald.conf | grep -E ^\#?Storage | cut -d"=" -f2` on one of your nodes. If `persistent` is returned your `systemdLogPath` should be `/var/log/journal`. If `volatile` is returned `systemdLogPath` should be `/run/log/journal`. If `auto` is returned check if `/var/log/journal` exists, and if it does then use `/var/log/journal`, otherwise use `/run/log/journal`. + +If any value not described here is returned, Rancher Logging will not be able to collect control plane logs. To address this issue set `Storage=volatile` in journald.conf, reboot your machine, and set `systemdLogPath` to `/run/log/journal`. diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/NOTES.txt b/charts/rancher-logging/104.2.0+up4.8.0/templates/NOTES.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_fluentbitagent.yaml new file mode 100644 index 0000000000..44f2303db3 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_fluentbitagent.yaml @@ -0,0 +1,85 @@ +{{- define "logging-operator.fluentbitagent.tpl" -}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: FluentbitAgent +metadata: + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + image: + repository: {{ template "logging-operator.fluentbitImageRepository" . }} + tag: {{ template "logging-operator.fluentbitImageTag" . }} + {{- if not .Values.disablePvc }} + {{- with .Values.fluentbit.bufferStorage }} + bufferStorage: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.bufferStorageVolume }} + bufferStorageVolume: {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- if or .Values.global.cattle.psp.enabled .Values.global.seLinux.enabled }} + security: + {{- end }} + {{- if .Values.global.cattle.psp.enabled }} + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + {{- if or .Values.fluentbit.inputTail.Buffer_Chunk_Size .Values.fluentbit.inputTail.Buffer_Max_Size .Values.fluentbit.inputTail.Mem_Buf_Limit .Values.fluentbit.inputTail.Multiline_Flush .Values.fluentbit.inputTail.Skip_Long_Lines }} + inputTail: + {{- if .Values.fluentbit.inputTail.Buffer_Chunk_Size }} + Buffer_Chunk_Size: {{ .Values.fluentbit.inputTail.Buffer_Chunk_Size | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Buffer_Max_Size }} + Buffer_Max_Size: {{ .Values.fluentbit.inputTail.Buffer_Max_Size | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Mem_Buf_Limit }} + Mem_Buf_Limit: {{ .Values.fluentbit.inputTail.Mem_Buf_Limit | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Multiline_Flush }} + Multiline_Flush: {{ .Values.fluentbit.inputTail.Multiline_Flush | toString | quote }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Skip_Long_Lines }} + Skip_Long_Lines: {{ .Values.fluentbit.inputTail.Skip_Long_Lines | toString | quote }} + {{- end }} + {{- end }} + {{- with (concat (.Values.tolerations) (.Values.fluentbit.tolerations)) }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.metrics }} + metrics: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end -}} + +{{- define "logging-operator.util.merge.fluentbitagent" -}} +{{/* Top context to expose fields like `.Release` and `.Values` */}} +{{- $top := first . -}} + +{{/* tpl is the template specific to the fluentbit implementation */}} +{{- $tpl := fromYaml (include (index . 1) $top) | default (dict ) -}} + +{{/* Generic is the shared rancher fluentbit setttings from `_generic_fluentbitagent.yaml` */}} +{{- $generic := fromYaml (include (index . 2) $top) | default (dict ) -}} + +{{/* values are from the values.yaml */}} +{{- $values := $top.Values.fluentbitAgentOverlay | default (dict ) -}} + +####### {{$generic}} + +{{/* the sources are merge right to left meaning tpl is the highest prcedence and values is the lowest */}} +{{- toYaml (merge $tpl $values $generic) -}} +{{- end -}} + +{{- define "logging-operator.fluentbitagent" -}} +{{- include "logging-operator.util.merge.fluentbitagent" (append . "logging-operator.fluentbitagent.tpl") -}} +{{- end -}} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_logging.yaml new file mode 100644 index 0000000000..ee39ef9682 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/_generic_logging.yaml @@ -0,0 +1,75 @@ +{{- define "logging-operator.logging.tpl" -}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentd: + {{- with .Values.fluentd.logLevel }} + logLevel: {{ . }} + {{- end }} + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + {{- with .Values.fluentd.bufferStorageVolume }} + bufferStorageVolume: {{- toYaml . | nindent 6 }} + {{- end }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.fluentd.replicas }} + scaling: + replicas: {{ .Values.fluentd.replicas }} + {{- end }} + security: + podSecurityContext: + {{- if .Values.global.cattle.psp.enabled }} + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.fluentd.env }} + envVars: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with (default .Values.tolerations .Values.fluentd.tolerations) }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with (default .Values.nodeSelector .Values.fluentd.nodeSelector) }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.metrics }} + metrics: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end -}} + +{{- define "logging-operator.util.merge.logging" -}} +{{/* Top context to expose fields like `.Release` and `.Values` */}} +{{- $top := first . -}} + +{{/* tpl is the template specific to the logging implementation */}} +{{- $tpl := fromYaml (include (index . 1) $top) | default (dict ) -}} + +{{/* Generic is the shared rancher logging setttings from `_generic_logging.yaml` */}} +{{- $generic := fromYaml (include (index . 2) $top) | default (dict ) -}} + +{{/* values are from the values.yaml */}} +{{- $values := $top.Values.loggingOverlay | default (dict ) -}} + +####### {{$generic}} + +{{/* the sources are merge right to left meaning tpl is the highest prcedence and values is the lowest */}} +{{- toYaml (merge $tpl $values $generic) -}} +{{- end -}} + +{{- define "logging-operator.logging" -}} +{{- include "logging-operator.util.merge.logging" (append . "logging-operator.logging.tpl") -}} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/_helpers.tpl b/charts/rancher-logging/104.2.0+up4.8.0/templates/_helpers.tpl new file mode 100644 index 0000000000..98ffa95685 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/_helpers.tpl @@ -0,0 +1,231 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "logging-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "logging-operator.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Expand the name of the release. +*/}} +{{- define "logging-operator.releasename" -}} +{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Provides the namespace the chart will be installed in using the builtin .Release.Namespace, +or, if provided, a manually overwritten namespace value. +*/}} +{{- define "logging-operator.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{ .Values.namespaceOverride -}} +{{- else -}} +{{ .Release.Namespace }} +{{- end -}} +{{- end -}} + + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "logging-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "logging-operator.labels" -}} +app.kubernetes.io/name: {{ include "logging-operator.name" . }} +helm.sh/chart: {{ include "logging-operator.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{- define "windowsEnabled" }} +{{- if not (kindIs "invalid" .Values.global.cattle.windows) }} +{{- if not (kindIs "invalid" .Values.global.cattle.windows.enabled) }} +{{- if .Values.global.cattle.windows.enabled }} +true +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "windowsPathPrefix" -}} +{{- trimSuffix "/" (default "c:\\" .Values.global.cattle.rkeWindowsPathPrefix | replace "\\" "/" | replace "//" "/" | replace "c:" "C:") -}} +{{- end -}} + +{{- define "windowsKubernetesFilter" -}} +{{- printf "kubernetes.%s" ((include "windowsPathPrefix" .) | replace ":" "" | replace "/" ".") -}} +{{- end -}} + +{{- define "windowsInputTailMount" -}} +{{- (include "windowsPathPrefix" .) | replace "C:" "" -}} +{{- end -}} + +{{/* +Set the controlplane selector based on kubernetes distribution +*/}} +{{- define "controlplaneSelector" -}} +{{- $master := or .Values.additionalLoggingSources.rke2.enabled .Values.additionalLoggingSources.k3s.enabled -}} +{{- $defaultSelector := $master | ternary (dict "node-role.kubernetes.io/master" "true") (dict "node-role.kubernetes.io/controlplane" "true") -}} +{{ default $defaultSelector .Values.additionalLoggingSources.kubeAudit.nodeSelector | toYaml }} +{{- end -}} + +{{/* +Set kube-audit file path prefix based on distribution +*/}} +{{- define "kubeAuditPathPrefix" -}} +{{- if .Values.additionalLoggingSources.rke.enabled -}} +{{ default "/var/log/kube-audit" .Values.additionalLoggingSources.kubeAudit.pathPrefix }} +{{- else if .Values.additionalLoggingSources.rke2.enabled -}} +{{ default "/var/lib/rancher/rke2/server/logs" .Values.additionalLoggingSources.kubeAudit.pathPrefix }} +{{- else -}} +{{ required "Directory PathPrefix of the kube-audit location is required" .Values.additionalLoggingSources.kubeAudit.pathPrefix }} +{{- end -}} +{{- end -}} + +{{/* +Set kube-audit file name based on distribution +*/}} +{{- define "kubeAuditFilename" -}} +{{- if .Values.additionalLoggingSources.rke.enabled -}} +{{ default "audit-log.json" .Values.additionalLoggingSources.kubeAudit.auditFilename }} +{{- else if .Values.additionalLoggingSources.rke2.enabled -}} +{{ default "audit.log" .Values.additionalLoggingSources.kubeAudit.auditFilename }} +{{- else -}} +{{ required "Filename of the kube-audit log is required" .Values.additionalLoggingSources.kubeAudit.auditFilename }} +{{- end -}} +{{- end -}} + +{{/* +A shared list of custom parsers for the vairous fluentbit pods rancher creates +*/}} +{{- define "logging-operator.parsers" -}} +[PARSER] + Name klog + Format regex + Regex ^(?[IWEF])(?\d{4} \d{2}:\d{2}:\d{2}).\d{6} +?(?\d+) (?.+):(?\d+)] (?.+) + Time_Key timestamp + Time_Format %m%d %T + +[PARSER] + Name rancher + Format regex + Regex ^time="(?.+)" level=(?.+) msg="(?.+)"$ + Time_Key timestamp + Time_Format %FT%H:%M:%S +[PARSER] + Name etcd + Format json + Time_Key timestamp + Time_Format %FT%H:%M:%S.%L +{{- end -}} + +{{/* +Set kubernetes log options if they are configured +*/}} +{{- define "requireFilterKubernetes" -}} +{{- if or .Values.fluentbit.filterKubernetes.Merge_Log .Values.fluentbit.filterKubernetes.Merge_Log_Key .Values.fluentbit.filterKubernetes.Merge_Trim .Values.fluentbit.filterKubernetes.Merge_Parser -}} +true +{{- end -}} +{{- end -}} + +{{/*Fluent Bit Image Repository */}} +{{- define "logging-operator.fluentbitImageRepository" -}} +{{- if .Values.debug -}} +{{ template "system_default_registry" . }}{{ .Values.images.fluentbit_debug.repository }} +{{- else -}} +{{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} +{{- end -}} +{{- end -}} + +{{/*Fluent Bit Image Tag */}} +{{- define "logging-operator.fluentbitImageTag" -}} +{{- if .Values.debug -}} +{{ .Values.images.fluentbit_debug.tag }} +{{- else -}} +{{ .Values.images.fluentbit.tag }} +{{- end -}} +{{- end -}} + +{{/*Fluent Bit Image */}} +{{- define "logging-operator.fluentbitImage" -}} +{{ template "logging-operator.fluentbitImageRepository" . }}:{{ template "logging-operator.fluentbitImageTag" . }} +{{- end -}} + +{{/* +Formats the cluster domain as a suffix, e.g.: +.Values.clusterDomain == "", returns "" +.Values.clusterDomain == "cluster.local.", returns ".cluster.local." +*/}} +{{- define "logging-operator.clusterDomainAsSuffix" -}} +{{- if .Values.clusterDomain -}} +{{- printf ".%s" .Values.clusterDomain -}} +{{- end -}} +{{- end -}} + +{{/* Implements logic to add the loggingRef field to custom loggings based on the cluster type */}} +{{- define "logging-operator.individualLoggingRef" -}} +{{- with .loggingRef -}} +loggingRef: {{ . }} +{{- end -}} +{{- end -}} + +{{/* Implements logic to add fluentd spec fields to custom loggings based on the cluster type */}} +{{- define "logging-operator.individualFluentd" -}} +{{- if .fluentd -}} +{{- if .fluentd.scaling -}} +scaling: + replicas: {{ .fluentd.scaling.replicas }} +{{- end }} +{{- with .fluentd.resources }} +resources: {{ toYaml . | nindent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Implements logic to add fluentbit loggingRef field to custom loggings based on the cluster type */}} +{{- define "logging-operator.individualFluentbitLoggingRef" -}} +{{- with .loggingRef -}} +loggingRef: {{ . }} +{{- end -}} +{{- end -}} + +{{/* Implements logic to add fluentbit spec fields to custom fluentBitAgents based on the cluster type */}} +{{- define "logging-operator.individualFluentbit" -}} +{{- with .resources }} +resources: {{ toYaml . | nindent 2 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrole.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrole.yaml new file mode 100644 index 0000000000..1e2652a7ec --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrole.yaml @@ -0,0 +1,388 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "logging-operator.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + - namespaces + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - pods + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + - extensions + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + - extensions + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - get + - list + - watch +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - use + - watch +- apiGroups: + - logging-extensions.banzaicloud.io + resources: + - eventtailers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging-extensions.banzaicloud.io + resources: + - eventtailers/status + verbs: + - get + - patch + - update +- apiGroups: + - logging-extensions.banzaicloud.io + resources: + - hosttailers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging-extensions.banzaicloud.io + resources: + - hosttailers/status + verbs: + - get + - patch + - update +- apiGroups: + - logging.banzaicloud.io + resources: + - clusterflows + - clusteroutputs + - flows + - fluentbitagents + - fluentdconfigs + - loggings + - nodeagents + - outputs + - syslogngconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - clusterflows/status + - clusteroutputs/status + - flows/status + - fluentbitagents/status + - fluentdconfigs/status + - loggings/status + - nodeagents/status + - outputs/status + - syslogngconfigs/status + verbs: + - get + - patch + - update +- apiGroups: + - logging.banzaicloud.io + resources: + - loggingroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - loggingroutes/status + verbs: + - get + - patch + - update +- apiGroups: + - logging.banzaicloud.io + resources: + - loggings/finalizers + verbs: + - update +- apiGroups: + - logging.banzaicloud.io + resources: + - syslogngclusterflows + - syslogngclusteroutputs + - syslogngflows + - syslogngoutputs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - syslogngclusterflows/status + - syslogngclusteroutputs/status + - syslogngflows/status + - syslogngoutputs/status + verbs: + - get + - patch + - update +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrolebinding.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..89d17d094f --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "logging-operator.fullname" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +subjects: + - kind: ServiceAccount + name: {{ template "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "logging-operator.fullname" . }} + + {{- end }} \ No newline at end of file diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/crds.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/crds.yaml new file mode 100644 index 0000000000..f573652d04 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/crds.yaml @@ -0,0 +1,6 @@ +{{- if .Values.createCustomResource -}} +{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} +{{ $.Files.Get $path }} +--- +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/deployment.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/deployment.yaml new file mode 100644 index 0000000000..0cdf494e97 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.podLabels }} + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + args: + {{- range .Values.extraArgs }} + - {{ . }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + ports: + - name: http + containerPort: {{ .Values.http.port }} + {{- with .Values.env }} + env: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.securityContext }} + securityContext: {{ toYaml .Values.securityContext | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: {{ toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.rbac.serviceAccountName }} + serviceAccountName: {{ .Values.rbac.serviceAccountName }} + {{- else if .Values.rbac.enabled }} + serviceAccountName: {{ include "logging-operator.fullname" . }} + {{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/extra-manifests.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusterflows.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusterflows.yaml new file mode 100644 index 0000000000..3a1a46bee5 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusterflows.yaml @@ -0,0 +1,14 @@ +{{ if .Values.logging.enabled -}} +{{- range $clusterflow := .Values.logging.clusterFlows }} +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: {{ $clusterflow.name }} + namespace: {{ $.Values.logging.controlNamespace | default $.Release.Namespace }} + labels: +{{ include "logging-operator.labels" $ | indent 4 }} +spec: +{{ toYaml $clusterflow.spec | indent 2 }} +{{- end -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusteroutputs.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusteroutputs.yaml new file mode 100644 index 0000000000..60d9f30ae7 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/clusteroutputs.yaml @@ -0,0 +1,14 @@ +{{ if .Values.logging.enabled -}} +{{- range $clusteroutput := .Values.logging.clusterOutputs }} +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: {{ $clusteroutput.name }} + namespace: {{ $.Values.logging.controlNamespace | default $.Release.Namespace }} + labels: +{{ include "logging-operator.labels" $ | indent 4 }} +spec: +{{ toYaml $clusteroutput.spec | indent 2 }} +{{- end -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/eventtailer.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/eventtailer.yaml new file mode 100644 index 0000000000..830cf9b15d --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/eventtailer.yaml @@ -0,0 +1,41 @@ +{{- with $.Values.logging.eventTailer }} +{{- if and $.Values.logging.enabled .enabled }} +apiVersion: logging-extensions.banzaicloud.io/v1alpha1 +kind: EventTailer +metadata: + name: {{ .name }} +spec: + controlNamespace: {{ $.Values.logging.controlNamespace | default $.Release.Namespace }} + {{- with .image }} + image: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .pvc }} + {{- if .enabled }} + positionVolume: + pvc: + spec: + accessModes: {{ .accessModes }} + resources: + requests: + storage: {{ .storage }} + volumeMode: {{ .volumeMode }} + {{- with .storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- end }}{{/* end if enabled */}} + {{- end }}{{/* end with pvc */}} + {{- with .workloadMetaOverrides }} + workloadMetaOverrides: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .workloadOverrides }} + workloadOverrides: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .containerOverrides }} + containerOverrides: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }}{{/* end if enabled */}} +{{- end }}{{/* end with event-tailer */}} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/fluentbit.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/fluentbit.yaml new file mode 100644 index 0000000000..ad9bdcb319 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/fluentbit.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.logging.enabled (not .Values.logging.fluentbitDisabled) -}} +{{- $fluentbitSpec := .Values.logging.fluentbit }} +{{- if .Values.logging.loggingRef }} + {{- $fluentbitSpec := set .Values.logging.fluentbit "loggingRef" (default .Values.logging.loggingRef .Values.logging.fluentbit.loggingRef) -}} +{{- end }} + +apiVersion: logging.banzaicloud.io/v1beta1 +kind: FluentbitAgent +metadata: + name: {{ include "logging-operator.releasename" . }} + labels: {{ include "logging-operator.labels" . | nindent 4 }} +{{- if $fluentbitSpec }} +spec: {{- toYaml $fluentbitSpec | nindent 2 }} +{{- else }} +spec: {} +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/hosttailer.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/hosttailer.yaml new file mode 100644 index 0000000000..a14d7c10b7 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/hosttailer.yaml @@ -0,0 +1,31 @@ +{{- with .Values.logging.hostTailer }} +{{- if and $.Values.logging.enabled .enabled }} +--- +apiVersion: logging-extensions.banzaicloud.io/v1alpha1 +kind: HostTailer +metadata: + name: {{ .name }} +spec: + {{- with .fileTailers }} + fileTailers: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .systemdTailers }} + systemdTailers: + {{- toYaml . | nindent 4 }} + {{- end }} + enableRecreateWorkloadOnImmutableFieldChange: {{ $.Values.logging.enableRecreateWorkloadOnImmutableFieldChange }} + {{- with .workloadMetaOverrides }} + workloadMetaOverrides: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .workloadOverrides }} + workloadOverrides: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .image }} + image: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/logging.yaml new file mode 100644 index 0000000000..0adf992bfc --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/logging/logging.yaml @@ -0,0 +1,60 @@ +{{ if .Values.logging.enabled -}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ include "logging-operator.releasename" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + {{- with .Values.logging.loggingRef }} + loggingRef: {{ . }} + {{- end }} + {{- with .Values.logging.flowConfigCheckDisabled }} + flowConfigCheckDisabled: {{ . }} + {{- end }} + {{- with .Values.logging.skipInvalidResources }} + skipInvalidResources: {{ . }} + {{- end }} + {{- with .Values.logging.flowConfigOverride }} + flowConfigOverride: {{ . }} + {{- end }} + {{- if (not .Values.logging.fluentdDisabled) }} + {{- if .Values.logging.fluentd }} + fluentd: {{- toYaml .Values.logging.fluentd | nindent 4 }} + {{- else }} + fluentd: {} + {{- end }} + {{- end }} + {{- with .Values.logging.syslogNG }} + syslogNG: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logging.defaultFlow }} + defaultFlow: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logging.errorOutputRef }} + errorOutputRef: {{ . }} + {{- end }} + {{- with .Values.logging.globalFilters }} + globalFilters: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logging.watchNamespaces }} + watchNamespaces: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logging.watchNamespaceSelector }} + watchNamespaceSelector: {{- toYaml . | nindent 4 }} + {{- end }} + clusterDomain: {{ .Values.logging.clusterDomain }} + controlNamespace: {{ .Values.logging.controlNamespace | default .Release.Namespace }} + {{- with .Values.logging.allowClusterResourcesFromAllNamespaces }} + allowClusterResourcesFromAllNamespaces: {{ . }} + {{- end }} + {{- with .Values.logging.nodeAgents }} + nodeAgents: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.logging.enableRecreateWorkloadOnImmutableFieldChange }} + enableRecreateWorkloadOnImmutableFieldChange: {{ . }} + {{- end }} + {{- with .Values.logging.configCheck }} + configCheck: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/fluentbitagent.yaml new file mode 100644 index 0000000000..8f06a2ed4e --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/fluentbitagent.yaml @@ -0,0 +1,20 @@ +{{- define "logging-operator.fluentbitagent.aks" -}} +{{- $logPath := "/var/log/azure/kubelet-status.log" -}} +{{- $individualValues := .Values.additionalLoggingSources.aks.fluentbit -}} +metadata: + name: {{ .Release.Name }}-aks +spec: + disableKubernetesFilter: true + extraVolumeMounts: + - source: {{ $logPath }} + destination: {{ $logPath }} + readOnly: true + inputTail: + Tag: "aks" + Path: {{ $logPath }} +{{- include "logging-operator.individualFluentbitLoggingRef" $individualValues | nindent 2 }} +{{- include "logging-operator.individualFluentbit" $individualValues | nindent 2 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.aks.enabled }} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.aks") -}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/logging.yaml new file mode 100644 index 0000000000..abed474351 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/aks/logging.yaml @@ -0,0 +1,12 @@ +{{- define "logging-operator.logging.aks" -}} +{{- $individualValues := .Values.additionalLoggingSources.aks -}} +metadata: + name: {{ .Release.Name }}-aks +spec: + {{- include "logging-operator.individualLoggingRef" $individualValues | nindent 2 }} + fluentd: + {{- include "logging-operator.individualFluentd" $individualValues | nindent 4 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.aks.enabled }} +{{- include "logging-operator.logging" (list . "logging-operator.logging.aks") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/fluentbitagent.yaml new file mode 100644 index 0000000000..cf0ca31336 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/fluentbitagent.yaml @@ -0,0 +1,21 @@ +{{- define "logging-operator.fluentbitagent.eks" -}} +{{- $logPath := "/var/log/messages" -}} +{{- $individualValues := .Values.additionalLoggingSources.eks.fluentbit -}} +metadata: + name: {{ .Release.Name }}-eks +spec: + {{- include "logging-operator.individualFluentbitLoggingRef" $individualValues | nindent 2 }} + disableKubernetesFilter: true + extraVolumeMounts: + - source: {{ $logPath }} + destination: {{ $logPath }} + readOnly: true + inputTail: + Tag: "eks" + Path: {{ $logPath }} + Parser: "syslog" + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 2 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.eks.enabled }} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.eks") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/logging.yaml new file mode 100644 index 0000000000..830d8ec390 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/eks/logging.yaml @@ -0,0 +1,20 @@ +{{- define "logging-operator.logging.eks" -}} +{{- $individualValues := .Values.additionalLoggingSources.eks -}} +metadata: + name: {{ .Release.Name }}-eks +spec: + {{- include "logging-operator.individualLoggingRef" $individualValues | nindent 2 }} + fluentd: + {{- include "logging-operator.individualFluentd" $individualValues | nindent 4 }} + {{- if .Values.loggingServiceAccountAnnotations.eks -}} + serviceAccount: + metadata: + annotations: + {{- with .Values.loggingServiceAccountAnnotations.eks }} + {{ toYaml . | indent 8 }} + {{- end }} + {{- end }} +{{- end -}} +{{- if .Values.additionalLoggingSources.eks.enabled }} +{{- include "logging-operator.logging" (list . "logging-operator.logging.eks") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/fluentbitagent.yaml new file mode 100644 index 0000000000..7541332404 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/fluentbitagent.yaml @@ -0,0 +1,20 @@ +{{- define "logging-operator.fluentbitagent.gke" -}} +{{- $logPath := "/var/log/kube-proxy.log" -}} +{{- $individualValues := .Values.additionalLoggingSources.gke.fluentbit -}} +metadata: + name: {{ .Release.Name }}-gke +spec: + {{- include "logging-operator.individualFluentbitLoggingRef" $individualValues | nindent 2 }} + disableKubernetesFilter: true + extraVolumeMounts: + - source: {{ $logPath }} + destination: {{ $logPath }} + readOnly: true + inputTail: + Tag: "gke" + Path: {{ $logPath }} + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 2 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.gke.enabled }} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.gke") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/logging.yaml new file mode 100644 index 0000000000..80ed58e280 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/gke/logging.yaml @@ -0,0 +1,12 @@ +{{- define "logging-operator.logging.gke" -}} +{{- $individualValues := .Values.additionalLoggingSources.gke -}} +metadata: + name: {{ .Release.Name }}-gke +spec: + {{- include "logging-operator.individualLoggingRef" $individualValues | nindent 2 }} + fluentd: + {{- include "logging-operator.individualFluentd" $individualValues | nindent 4 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.gke.enabled }} +{{- include "logging-operator.logging" (list . "logging-operator.logging.gke") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/configmap.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/configmap.yaml new file mode 100644 index 0000000000..aa454c8adf --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/configmap.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "systemd") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-k3s + labels: +{{ include "logging-operator.labels" . | indent 4 }} +data: + fluent-bit.conf: | + [SERVICE] + Flush 1 + Grace 5 + Daemon Off + Log_Level info + Coro_Stack_Size 24576 + Parsers_File parsers.conf + + [INPUT] + Name systemd + Tag k3s + Path {{ .Values.systemdLogPath }} + Systemd_Filter _SYSTEMD_UNIT=k3s.service + {{- if .Values.additionalLoggingSources.k3s.stripUnderscores }} + Strip_Underscores On + {{- end }} + Systemd_Filter _SYSTEMD_UNIT=k3s-agent.service + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser klog + Reserve_Data On + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser rancher + Reserve_Data On + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser etcd + Reserve_Data On + + [OUTPUT] + Name forward + Match * + Host {{ .Release.Name }}-root-fluentd.{{ .Release.Namespace }}.svc + Port 24240 + Retry_Limit False + parsers.conf: | +{{ include "logging-operator.parsers" . | indent 4 }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/daemonset.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/daemonset.yaml new file mode 100644 index 0000000000..4700873fe7 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/daemonset.yaml @@ -0,0 +1,112 @@ +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "systemd") }} +{{- $individualValues := .Values.additionalLoggingSources.k3s.fluentbit -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: "{{ .Release.Name }}-k3s-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-k3s-journald-aggregator + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/loggings/k3s/configmap.yaml") . | sha256sum }} + name: "{{ .Release.Name }}-k3s-journald-aggregator" + namespace: "{{ .Release.Namespace }}" + labels: + name: {{ .Release.Name }}-k3s-journald-aggregator + spec: + containers: + - name: fluentbit + image: "{{ template "logging-operator.fluentbitImage" . }}" + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 10 }} + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + volumeMounts: + - mountPath: /fluent-bit/etc/ + name: config + - mountPath: {{ .Values.systemdLogPath | default "/var/log/journal" }} + name: journal + readOnly: true + - mountPath: /etc/machine-id + name: machine-id + readOnly: true + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: "{{ .Release.Name }}-k3s-journald-aggregator" + volumes: + - name: config + configMap: + name: "{{ .Release.Name }}-k3s" + - name: journal + hostPath: + path: {{ .Values.systemdLogPath | default "/var/log/journal" }} + - name: machine-id + hostPath: + path: /etc/machine-id +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Release.Name }}-k3s-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +{{- if .Values.global.cattle.psp.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Release.Name }}-k3s-journald-aggregator" +rules: + - apiGroups: + - policy + resourceNames: + - "{{ .Release.Name }}-k3s-journald-aggregator" + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ .Release.Name }}-k3s-journald-aggregator" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ .Release.Name }}-k3s-journald-aggregator" +subjects: + - kind: ServiceAccount + name: "{{ .Release.Name }}-k3s-journald-aggregator" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: "{{ .Release.Name }}-k3s-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + allowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + readOnlyRootFilesystem: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - emptyDir + - secret + - hostPath +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/fluentbitagent.yaml new file mode 100644 index 0000000000..9bde882ad0 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/fluentbitagent.yaml @@ -0,0 +1,21 @@ +{{- define "logging-operator.fluentbitagent.k3s-openrc" -}} +{{- $logPath := "/var/log/k3s.log" -}} +{{- $individualValues := .Values.additionalLoggingSources.k3s.fluentbit -}} +metadata: + name: {{ .Release.Name }}-k3s +spec: + {{- include "logging-operator.individualFluentbitLoggingRef" $individualValues | nindent 2 }} + disableKubernetesFilter: true + extraVolumeMounts: + - source: {{ $logPath }} + destination: {{ $logPath }} + readOnly: true + inputTail: + Tag: "k3s" + Path: {{ $logPath }} + Path_Key: filename + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 2 }} +{{- end -}} +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "openrc")}} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.k3s-openrc") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/logging-k3s-openrc.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/logging-k3s-openrc.yaml new file mode 100644 index 0000000000..6d91c33471 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/k3s/logging-k3s-openrc.yaml @@ -0,0 +1,12 @@ +{{- define "logging-operator.logging.k3s-openrc" -}} +{{- $individualValues := .Values.additionalLoggingSources.k3s -}} +metadata: + name: {{ .Release.Name }}-k3s + spec: + {{- include "logging-operator.individualLoggingRef" $individualValues | nindent 2 }} + fluentd: + {{- include "logging-operator.individualFluentd" $individualValues | nindent 4 }} +{{- end -}} +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "openrc")}} +{{- include "logging-operator.logging" (list . "logging-operator.logging.k3s-openrc") -}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/fluentbitagent.yaml new file mode 100644 index 0000000000..c0e17a71b5 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/fluentbitagent.yaml @@ -0,0 +1,24 @@ +{{- define "logging-operator.fluentbitagent.kube-audit" -}} +{{- $individualValues := .Values.additionalLoggingSources.kubeAudit.fluentbit -}} +metadata: + name: {{ .Release.Name }}-kube-audit +spec: + {{- include "logging-operator.individualFluentbitLoggingRef" $individualValues | nindent 2 }} + disableKubernetesFilter: true + extraVolumeMounts: + - source: {{ template "kubeAuditPathPrefix" . }} + destination: "/kube-audit-logs" + readOnly: true + inputTail: + Tag: {{ .Values.additionalLoggingSources.kubeAudit.fluentbit.logTag }} + Path: /kube-audit-logs/{{ template "kubeAuditFilename" . }} + Parser: json + {{- with (concat (.Values.tolerations) (.Values.fluentbit.tolerations) (.Values.additionalLoggingSources.kubeAudit.fluentbit.tolerations)) }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + nodeSelector: {{ include "controlplaneSelector" . | nindent 6 }} + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 2 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.kubeAudit.enabled }} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.kube-audit") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/logging.yaml new file mode 100644 index 0000000000..b2e5fc24c2 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/kube-audit/logging.yaml @@ -0,0 +1,15 @@ +{{- define "logging-operator.logging.kube-audit" -}} +{{- $individualValues := .Values.additionalLoggingSources.kubeAudit -}} +metadata: + name: {{ .Release.Name }}-kube-audit +spec: + {{- include "logging-operator.individualLoggingRef" $individualValues | nindent 2 }} + {{- if .Values.additionalLoggingSources.kubeAudit.loggingRef }} + loggingRef: {{ .Values.additionalLoggingSources.kubeAudit.loggingRef }} + {{- end }} + fluentd: + {{- include "logging-operator.individualFluentd" $individualValues | nindent 4 }} +{{- end -}} +{{- if .Values.additionalLoggingSources.kubeAudit.enabled }} +{{- include "logging-operator.logging" (list . "logging-operator.logging.kube-audit") -}} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/configmap.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/configmap.yaml new file mode 100644 index 0000000000..252572a4ef --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/configmap.yaml @@ -0,0 +1,29 @@ +{{- if .Values.additionalLoggingSources.rke.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-rke + labels: +{{ include "logging-operator.labels" . | indent 4 }} +data: + fluent-bit.conf: | + [SERVICE] + Log_Level {{ .Values.additionalLoggingSources.rke.fluentbit.log_level }} + Parsers_File parsers.conf + + [INPUT] + Tag rke + Name tail + Path_Key filename + Parser docker + DB /tail-db/tail-containers-state.db + Mem_Buf_Limit {{ .Values.additionalLoggingSources.rke.fluentbit.mem_buffer_limit }} + Path /var/lib/rancher/rke/log/*.log + + [OUTPUT] + Name forward + Match * + Host {{ .Release.Name }}-root-fluentd.{{ .Release.Namespace }}.svc + Port 24240 + Retry_Limit False +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/daemonset.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/daemonset.yaml new file mode 100644 index 0000000000..b050e8d17e --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke/daemonset.yaml @@ -0,0 +1,124 @@ +{{- if .Values.additionalLoggingSources.rke.enabled }} +{{- $containers := printf "%s/containers/" (default "/var/lib/docker" .Values.global.dockerRootDirectory) }} +{{- $individualValues := .Values.additionalLoggingSources.rke.fluentbit -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-rke-aggregator + template: + metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" + labels: + name: {{ .Release.Name }}-rke-aggregator + spec: + containers: + - name: fluentbit + image: "{{ template "logging-operator.fluentbitImage" . }}" + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 10 }} + volumeMounts: + - mountPath: /var/lib/rancher/rke/log/ + name: indir + - mountPath: {{ $containers }} + name: containers + - mountPath: /tail-db + name: positiondb + - mountPath: /fluent-bit/etc/fluent-bit.conf + name: config + subPath: fluent-bit.conf + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + volumes: + - name: indir + hostPath: + path: /var/lib/rancher/rke/log/ + type: DirectoryOrCreate + - name: containers + hostPath: + path: {{ $containers }} + type: DirectoryOrCreate + - name: positiondb + emptyDir: {} + - name: config + configMap: + name: "{{ .Release.Name }}-rke" + serviceAccountName: "{{ .Release.Name }}-rke-aggregator" + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +{{- if .Values.global.cattle.psp.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Release.Name }}-rke-aggregator" +rules: + - apiGroups: + - policy + resourceNames: + - "{{ .Release.Name }}-rke-aggregator" + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ .Release.Name }}-rke-aggregator" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ .Release.Name }}-rke-aggregator" +subjects: + - kind: ServiceAccount + name: "{{ .Release.Name }}-rke-aggregator" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + allowPrivilegeEscalation: false + allowedHostPaths: + - pathPrefix: {{ $containers }} + readOnly: false + - pathPrefix: /var/lib/rancher/rke/log/ + readOnly: false + - pathPrefix: /var/lib/rancher/logging/ + readOnly: false + fsGroup: + rule: RunAsAny + readOnlyRootFilesystem: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - emptyDir + - secret + - hostPath +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/configmap.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/configmap.yaml new file mode 100644 index 0000000000..3ca20be226 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/configmap.yaml @@ -0,0 +1,69 @@ +{{- if .Values.additionalLoggingSources.rke2.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-rke2 + labels: +{{ include "logging-operator.labels" . | indent 4 }} +data: + fluent-bit.conf: | + [SERVICE] + Flush 1 + Grace 5 + Daemon Off + Log_Level info + Coro_Stack_Size 24576 + Parsers_File parsers.conf + + [INPUT] + Name systemd + Tag rke2 + Path {{ .Values.systemdLogPath }} + Systemd_Filter _SYSTEMD_UNIT=rke2-server.service + Systemd_Filter _SYSTEMD_UNIT=rke2-agent.service + {{- if .Values.additionalLoggingSources.rke2.stripUnderscores }} + Strip_Underscores On + {{- end }} + + [INPUT] + Name tail + Tag rke2 + Path /var/lib/rancher/rke2/agent/logs/kubelet.log + + [FILTER] + Name parser + Match * + Key_Name log + Parser klog + Reserve_Data On + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser klog + Reserve_Data On + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser rancher + Reserve_Data On + + [FILTER] + Name parser + Match * + Key_Name MESSAGE + Parser etcd + Reserve_Data On + + [OUTPUT] + Name forward + Match * + Host {{ .Release.Name }}-root-fluentd.{{ .Release.Namespace }}.svc + Port 24240 + Retry_Limit False + parsers.conf: | +{{ include "logging-operator.parsers" . | indent 4 }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/daemonset.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/daemonset.yaml new file mode 100644 index 0000000000..37f3b353a1 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/rke2/daemonset.yaml @@ -0,0 +1,118 @@ +{{- if .Values.additionalLoggingSources.rke2.enabled }} +{{- $individualValues := .Values.additionalLoggingSources.rke2.fluentbit -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-rke2-journald-aggregator + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/loggings/rke2/configmap.yaml") . | sha256sum }} + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" + labels: + name: {{ .Release.Name }}-rke2-journald-aggregator + spec: + containers: + - name: fluentbit + image: "{{ template "logging-operator.fluentbitImage" . }}" + {{- include "logging-operator.individualFluentbit" $individualValues | nindent 10 }} + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + volumeMounts: + - mountPath: /fluent-bit/etc/ + name: config + - mountPath: {{ .Values.systemdLogPath | default "/var/log/journal" }} + name: journal + readOnly: true + - mountPath: "/var/lib/rancher/rke2/agent/logs" + name: kubelet + readOnly: true + - mountPath: /etc/machine-id + name: machine-id + readOnly: true + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: "{{ .Release.Name }}-rke2-journald-aggregator" + volumes: + - name: config + configMap: + name: "{{ .Release.Name }}-rke2" + - name: journal + hostPath: + path: {{ .Values.systemdLogPath | default "/var/log/journal" }} + - name: kubelet + hostPath: + path: "/var/lib/rancher/rke2/agent/logs" + - name: machine-id + hostPath: + path: /etc/machine-id +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +{{- if .Values.global.cattle.psp.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" +rules: + - apiGroups: + - policy + resourceNames: + - "{{ .Release.Name }}-rke2-journald-aggregator" + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ .Release.Name }}-rke2-journald-aggregator" +subjects: + - kind: ServiceAccount + name: "{{ .Release.Name }}-rke2-journald-aggregator" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + allowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + readOnlyRootFilesystem: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - emptyDir + - secret + - hostPath +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/fluentbitagent.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/fluentbitagent.yaml new file mode 100644 index 0000000000..2816b7f911 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/fluentbitagent.yaml @@ -0,0 +1,29 @@ +{{- define "logging-operator.fluentbitagent.root" -}} +{{- $containerLogPath := printf "%s/containers/" (default "/var/lib/docker" .Values.global.dockerRootDirectory) }} +metadata: + name: "{{ .Release.Name }}-root" +spec: + {{- if .Values.global.dockerRootDirectory }} + mountPath: {{ $containerLogPath }} + extraVolumeMounts: + - source: {{ $containerLogPath }} + destination: {{ $containerLogPath }} + readOnly: true + {{- end }} + {{- if (include "requireFilterKubernetes" .) }} + filterKubernetes: + {{- if .Values.fluentbit.filterKubernetes.Merge_Log }} + Merge_Log: "{{ .Values.fluentbit.filterKubernetes.Merge_Log }}" + {{- end }} + {{- if .Values.fluentbit.filterKubernetes.Merge_Log_Key }} + Merge_Log_Key: "{{ .Values.fluentbit.filterKubernetes.Merge_Log_Key }}" + {{- end }} + {{- if .Values.fluentbit.filterKubernetes.Merge_Log_Trim }} + Merge_Log_Trim: "{{ .Values.fluentbit.filterKubernetes.Merge_Log_Trim }}" + {{- end }} + {{- if .Values.fluentbit.filterKubernetes.Merge_Parser }} + Merge_Parser: "{{ .Values.fluentbit.filterKubernetes.Merge_Parser }}" + {{- end }} + {{- end }} +{{- end -}} +{{- include "logging-operator.fluentbitagent" (list . "logging-operator.fluentbitagent.root") -}} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/logging.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/logging.yaml new file mode 100644 index 0000000000..e225815e1c --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/loggings/root/logging.yaml @@ -0,0 +1,67 @@ +{{- define "logging-operator.logging.root" -}} +metadata: + name: "{{ .Release.Name }}-root" +spec: + {{- if (include "windowsEnabled" .) }} + nodeAgents: + - name: win-agent + profile: windows + nodeAgentFluentbit: + daemonSet: + spec: + template: + spec: + containers: + - image: {{ template "system_default_registry" . }}{{ .Values.images.nodeagent_fluentbit.repository }}:{{ .Values.images.nodeagent_fluentbit.tag }} + name: fluent-bit + tls: + enabled: {{ .Values.nodeAgents.tls.enabled | default false }} + {{- if .Values.additionalLoggingSources.rke.enabled }} + - name: win-agent-rke + profile: windows + nodeAgentFluentbit: + filterKubernetes: + Kube_Tag_Prefix: "{{ template "windowsKubernetesFilter" . }}.var.lib.rancher.rke.log." + inputTail: + Path: "{{ template "windowsPathPrefix" . }}/var/lib/rancher/rke/log" + {{- if .Values.fluentbit.inputTail.Buffer_Chunk_Size }} + Buffer_Chunk_Size: {{ .Values.fluentbit.inputTail.Buffer_Chunk_Size | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Buffer_Max_Size }} + Buffer_Max_Size: {{ .Values.fluentbit.inputTail.Buffer_Max_Size | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Mem_Buf_Limit }} + Mem_Buf_Limit: {{ .Values.fluentbit.inputTail.Mem_Buf_Limit | toString }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Multiline_Flush }} + Multiline_Flush: {{ .Values.fluentbit.inputTail.Multiline_Flush | toString | quote }} + {{- end }} + {{- if .Values.fluentbit.inputTail.Skip_Long_Lines }} + Skip_Long_Lines: {{ .Values.fluentbit.inputTail.Skip_Long_Lines | toString | quote }} + {{- end }} + extraVolumeMounts: + - source: "{{ template "windowsInputTailMount" . }}/var/lib/rancher/rke/log" + destination: "{{ template "windowsInputTailMount" . }}/var/lib/rancher/rke/log" + readOnly: true + daemonSet: + spec: + template: + spec: + containers: + - image: "{{ template "system_default_registry" . }}{{ .Values.images.nodeagent_fluentbit.repository }}:{{ .Values.images.nodeagent_fluentbit.tag }}" + name: fluent-bit + tls: + enabled: {{ .Values.nodeAgents.tls.enabled | default false }} + {{- end }} + {{- end }} + fluentd: + {{- if .Values.loggingServiceAccountAnnotations.root }} + serviceAccount: + metadata: + annotations: + {{- with .Values.loggingServiceAccountAnnotations.root }} + {{ toYaml . | indent 8 }} + {{- end }} + {{- end -}} +{{- end -}} +{{- include "logging-operator.logging" (list . "logging-operator.logging.root") -}} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/service.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/service.yaml new file mode 100644 index 0000000000..f419ae2c48 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + type: ClusterIP + {{- with .Values.http.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - port: {{ .Values.http.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/service_monitor.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/service_monitor.yaml new file mode 100644 index 0000000000..fcd86b07a9 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/service_monitor.yaml @@ -0,0 +1,30 @@ +{{ if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.monitoring.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +{{- with .Values.monitoring.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} +{{- end }} +spec: + selector: + matchLabels: +{{ include "logging-operator.labels" . | indent 6 }} + endpoints: + - port: http + path: /metrics + {{- with .Values.monitoring.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.monitoring.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "logging-operator.namespace" . }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/serviceaccount.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/serviceaccount.yaml new file mode 100644 index 0000000000..bb97cf1084 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +{{- with .Values.serviceAccount.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/test_receiver.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/test_receiver.yaml new file mode 100644 index 0000000000..af0b5f4dcc --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/test_receiver.yaml @@ -0,0 +1,53 @@ +{{ if .Values.testReceiver.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "logging-operator.releasename" . }}-test-receiver + namespace: {{ include "logging-operator.namespace" . }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.testReceiver.port }} + targetPort: receiver + protocol: TCP + name: receiver + selector: + app.kubernetes.io/name: {{ include "logging-operator.releasename" . }}-test-receiver + app.kubernetes.io/instance: {{ .Release.Name }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "logging-operator.releasename" . }}-test-receiver + namespace: {{ include "logging-operator.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "logging-operator.releasename" . }}-test-receiver + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "logging-operator.releasename" . }}-test-receiver + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + fluentbit.io/exclude: "true" + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.testReceiver.image }}" + args: + {{- range .Values.testReceiver.args }} + - {{ . }} + {{- end }} + imagePullPolicy: {{ .Values.testReceiver.pullPolicy }} + resources: + {{- toYaml .Values.testReceiver.resources | nindent 12 }} + ports: + - name: receiver + containerPort: {{ .Values.testReceiver.port }} +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/userrole.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/userrole.yaml new file mode 100644 index 0000000000..82c3200b9c --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/userrole.yaml @@ -0,0 +1,39 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "logging-operator.fullname" . }}-edit + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: + - logging.banzaicloud.io + resources: + - flows + - outputs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - syslogngflows + - syslogngoutputs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/userroles.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/userroles.yaml new file mode 100644 index 0000000000..f4136b09a4 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/userroles.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "logging-admin" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: + - "logging.banzaicloud.io" + resources: + - flows + - outputs + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "logging-view" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: + - apiGroups: + - "logging.banzaicloud.io" + resources: + - flows + - outputs + - clusterflows + - clusteroutputs + verbs: + - get + - list + - watch diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install-crd.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..5184c85c47 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install-crd.yaml @@ -0,0 +1,29 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "logging-extensions.banzaicloud.io/v1alpha1/EventTailer" false -}} +# {{- set $found "logging-extensions.banzaicloud.io/v1alpha1/HostTailer" false -}} +# {{- set $found "logging.banzaicloud.io/v1alpha1/ClusterFlow" false -}} +# {{- set $found "logging.banzaicloud.io/v1alpha1/ClusterOutput" false -}} +# {{- set $found "logging.banzaicloud.io/v1alpha1/Flow" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/FluentbitAgent" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/FluentdConfig" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/LoggingRoute" false -}} +# {{- set $found "logging.banzaicloud.io/v1alpha1/Logging" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/NodeAgent" false -}} +# {{- set $found "logging.banzaicloud.io/v1alpha1/Output" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/SyslogNGClusterFlow" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/SyslogNGClusterOutput" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/SyslogNGConfig" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/SyslogNGFlow" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/SyslogNGOutput" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install.yaml new file mode 100644 index 0000000000..bd624cc4b4 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-install.yaml @@ -0,0 +1,5 @@ +#{{- if .Values.global.dockerRootDirectory }} +#{{- if or (hasSuffix "/containers" .Values.global.dockerRootDirectory) (hasSuffix "/" .Values.global.dockerRootDirectory) }} +#{{- required "global.dockerRootDirectory must not end with suffix: '/' or '/containers'" "" -}} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-psp-install.yaml b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-logging/104.2.0+up4.8.0/values-logging-example.yaml b/charts/rancher-logging/104.2.0+up4.8.0/values-logging-example.yaml new file mode 100644 index 0000000000..b7068af4a0 --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/values-logging-example.yaml @@ -0,0 +1,24 @@ +nameOverride: example + +# given we use `nameOverride: example` if testReceiver is enabled we can send http metrics to http://example-test-receiver:8080 +testReceiver: + enabled: true + +logging: + enabled: true + clusterFlows: + - name: all + spec: + match: + - select: {} + globalOutputRefs: ["http"] + clusterOutputs: + - name: http + spec: + http: + endpoint: http://example-test-receiver:8080 + content_type: application/json + buffer: + type: memory + timekey: 1s + timekey_wait: 1s diff --git a/charts/rancher-logging/104.2.0+up4.8.0/values.yaml b/charts/rancher-logging/104.2.0+up4.8.0/values.yaml new file mode 100644 index 0000000000..525181c54c --- /dev/null +++ b/charts/rancher-logging/104.2.0+up4.8.0/values.yaml @@ -0,0 +1,484 @@ +# Default values for logging-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rancher/mirrored-kube-logging-logging-operator + tag: 4.8.0 + pullPolicy: IfNotPresent + +env: [] +volumes: [] +volumeMounts: [] + +extraArgs: + - -enable-leader-election=true +imagePullSecrets: [] + +# -- A name in place of the chart name for `app:` labels. +nameOverride: "" + +# -- A name to substitute for the full names of resources. +fullnameOverride: "" + +# -- A namespace override for the app. +namespaceOverride: "" + +# -- Define annotations for logging-operator pods. +annotations: {} + +# -- Deploy CRDs used by Logging Operator. +createCustomResource: false + +http: + # -- HTTP listen port number. + port: 8080 + + # -- Service definition for query http service. + service: + type: ClusterIP + clusterIP: None + # Annotations to query http service + annotations: {} + # Labels to query http service + labels: {} + +rbac: + # -- Create rbac service account and roles. + enabled: true + + # specify service account manually + # serviceAccountName: custom + +monitoring: + serviceMonitor: + # -- Create a Prometheus Operator ServiceMonitor object. + enabled: false + + additionalLabels: {} + metricRelabelings: [] + relabelings: [] + +# -- Pod SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) +## SecurityContext holds pod-level security attributes and common container settings. +## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +podSecurityContext: {} +# runAsNonRoot: true +# runAsUser: 1000 +# fsGroup: 2000 + +# -- Container SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) +securityContext: {} +# allowPrivilegeEscalation: false +# readOnlyRootFilesystem: true + # capabilities: + # drop: ["ALL"] + +# -- Operator priorityClassName. +priorityClassName: {} + +serviceAccount: + # -- Define annotations for logging-operator ServiceAccount. + annotations: {} + +# -- CPU/Memory resource requests/limits +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +# -- Node Tolerations +tolerations: [] + +# -- Node Affinity +affinity: {} + +# -- Define which Nodes the Pods are scheduled on. +podLabels: {} + +# Logging resources configuration. +logging: + + # -- Logging resources are disabled by default + enabled: false + + # -- Reference to the logging system. Each of the loggingRefs can manage a fluentbit daemonset and a fluentd statefulset. + loggingRef: "" + + # -- Disable configuration check before applying new fluentd configuration. + flowConfigCheckDisabled: false + + # -- Whether to skip invalid Flow and ClusterFlow resources + skipInvalidResources: false + + # -- Override generated config. This is a raw configuration string for troubleshooting purposes. + flowConfigOverride: "" + + # -- Flag to disable fluentbit completely + fluentbitDisabled: false + # -- Fluent-bit configurations https://kube-logging.github.io/docs/configuration/crds/v1beta1/fluentbit_types/ + fluentbit: {} + + # -- Flag to disable fluentd completely + fluentdDisabled: false + # -- Fluentd configurations https://kube-logging.github.io/docs/configuration/crds/v1beta1/fluentd_types/ + fluentd: {} + # 20Gi persistent storage is configured for fluentd by default. + # Here is an example, on how to override it: + # bufferStorageVolume: + # pvc: + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 40Gi + + # -- Syslog-NG statefulset configuration + syslogNG: {} + + # -- Default flow for unmatched logs. This Flow configuration collects all logs that didn’t match any other Flow. + defaultFlow: {} + + # -- GlobalOutput name to flush ERROR events to + errorOutputRef: "" + + # -- Global filters to apply on logs before any match or filter mechanism. + globalFilters: [] + + # -- Limit namespaces to watch Flow and Output custom resources. + watchNamespaces: [] + + # -- Limit namespaces to watch Flow and Output custom resources. + watchNamespaceSelector: {} + + # -- Cluster domain name to be used when templating URLs to services + clusterDomain: "cluster.local." + + # -- Namespace for cluster wide configuration resources like ClusterFlow and ClusterOutput. This should be a protected namespace from regular users. Resources like fluentbit and fluentd will run in this namespace as well. + controlNamespace: "" + + # -- Allow configuration of cluster resources from any namespace. Mutually exclusive with ControlNamespace restriction of Cluster resources + allowClusterResourcesFromAllNamespaces: false + + # -- NodeAgent Configuration + nodeAgents: {} + # - name: win-agent + # profile: windows + # nodeAgentFluentbit: + # daemonSet: + # spec: + # template: + # spec: + # containers: + # - image: banzaicloud/fluentbit:1.9.5 + # name: fluent-bit + # tls: + # enabled: false + # - name: linux-agent + # profile: linux + # nodeAgentFluentbit: + # metrics: + # prometheusAnnotations: true + # serviceMonitor: false + # tls: + # enabled: false + + # -- configCheck provides possibility for timeout-based configuration checks https://kube-logging.dev/docs/whats-new/#timeout-based-configuration-checks + configCheck: {} + + # -- EnableRecreateWorkloadOnImmutableFieldChange enables the operator to recreate the fluentbit daemonset and the fluentd statefulset (and possibly other resource in the future) in case there is a change in an immutable field that otherwise couldn’t be managed with a simple update. + enableRecreateWorkloadOnImmutableFieldChange: false + + # -- ClusterFlows to deploy + clusterFlows: [] + + # -- ClusterOutputs to deploy + clusterOutputs: [] + + # Send all pod logs to kafka + # clusterFlows: + # - name: all + # spec: + # match: + # - select: {} + # globalOutputRefs: ["kafka"] + # clusterOutputs: + # - name: kafka + # spec: + # kafka: + # brokers: kafka-headless.kafka.svc.cluster.local:29092 + # format: + # type: json + # default_topic: topic + + # EventTailer config + eventTailer: + enabled: false + name: event-tailer + image: + # -- repository of eventTailer image + repository: + # -- tag of eventTailer image + tag: + # -- pullPolicy of eventTailer image + pullPolicy: + # -- imagePullSecrets of eventTailer image + imagePullSecrets: [] + pvc: + # -- enable pvc for + enabled: false + # -- storage class for event tailer pvc + accessModes: + - ReadWriteOnce + # -- storage class for event tailer pvc + volumeMode: Filesystem + # -- storage for event tailer pvc + storage: 1Gi + # -- storage class for event tailer pvc + storageClassName: + # -- workloadMetaOverrides + workloadMetaOverrides: + # -- workloadOverrides + workloadOverrides: + # -- containerOverrides + containerOverrides: + + hostTailer: + # -- HostTailer + enabled: false + # -- name of HostTailer + name: hosttailer + image: + # -- repository of eventTailer image + repository: + # -- tag of eventTailer image + tag: + # -- pullPolicy of eventTailer image + pullPolicy: + # -- imagePullSecrets of eventTailer image + imagePullSecrets: [] + # -- workloadMetaOverrides of HostTailer + workloadMetaOverrides: + # -- workloadOverrides of HostTailer + workloadOverrides: + # -- configure fileTailers of HostTailer + # example: + # - name: sample-file + # path: /var/log/sample-file + # disabled: false + # buffer_max_size: + # buffer_chunk_size: + # skip_long_lines: + # read_from_head: false + # containerOverrides: + # image: + fileTailers: [] + # -- configure systemdTailers of HostTailer + # example: + # - name: system-sample + # disabled: false + # systemdFilter: kubelet.service + # maxEntries: 20 + # containerOverrides: + # image: + systemdTailers: [] + +testReceiver: + enabled: false + image: fluent/fluent-bit + pullPolicy: IfNotPresent + port: 8080 + # args: ["-i", "http", "-p", "port=8080", "-o", "stdout"] + # resources: + # limits: + # cpu: 100m + # memory: 50Mi + # requests: + # cpu: 20m + # memory: 25Mi + + # Service definition for query http service + service: + type: ClusterIP + clusterIP: None + # Annotations to query http service + annotations: {} + # Labels to query http service + labels: {} + +# Logging CR specific serviceAccount annotations +loggingServiceAccountAnnotations: {} +## Syntax ## +# : +# : +# +## Example ## +# +# root: +# eks.amazonaws.com/role-arn: +# +## Result - added to the Logging resource ## +# +# spec: +# fluentd: +# serviceAccount: +# metadata: +# annotations: +# eks.amazonaws.com/role-arn: arn:aws:iam::1234567890:role/my-iam-role +# + +################################### +# Rancher Logging Operator Values # +################################### + +# Enable debug to use fluent-bit images that allow exec +debug: false + +# Disable persistent volumes for buffers +disablePvc: true + +# If your additional logging sources collect logs from systemd configure the systemd log path here +systemdLogPath: "/run/log/journal" + +global: + cattle: + systemDefaultRegistry: "" + # Uncomment the below two lines to either enable or disable Windows logging. If this chart is + # installed via the Rancher UI, it will set this value to "true" if the cluster is a Windows + # cluster. In that scenario, if you would like to disable Windows logging on Windows clusters, + # set the value below to "false". + # windows: + # enabled: true + psp: + enabled: false + + # Change the "dockerRootDirectory" if the default Docker directory has changed. + dockerRootDirectory: "" + + rkeWindowsPathPrefix: "c:\\" + + seLinux: + enabled: false + +images: + config_reloader: + repository: rancher/mirrored-kube-logging-config-reloader + tag: v0.0.5 + fluentbit: + repository: rancher/mirrored-fluent-fluent-bit + tag: 3.0.4 + nodeagent_fluentbit: + os: "windows" + repository: rancher/fluent-bit + tag: 2.2.0 + fluentbit_debug: + repository: rancher/mirrored-fluent-fluent-bit + tag: 3.0.4-debug + fluentd: + repository: rancher/mirrored-kube-logging-fluentd + tag: v1.16-4.8-full + +additionalLoggingSources: + rke: + enabled: false + fluentbit: + log_level: "info" + mem_buffer_limit: "5MB" + rke2: + enabled: false + stripUnderscores: false + k3s: + enabled: false + container_engine: "systemd" + stripUnderscores: false + aks: + enabled: false + eks: + enabled: false + gke: + enabled: false + kubeAudit: + auditFilename: "" + enabled: false + pathPrefix: "" + fluentbit: + logTag: kube-audit + tolerations: + - key: node-role.kubernetes.io/control-plane + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/etcd + value: "true" + effect: NoExecute + +# configures node agent options for windows node agents +nodeAgents: + tls: + enabled: false + +# These settings apply to every Logging CR, including vendor Logging CRs enabled in "additionalLoggingSources". +# Changing these affects every Logging CR installed. +fluentd: + bufferStorageVolume: {} + livenessProbe: + tcpSocket: + port: 24240 + initialDelaySeconds: 30 + periodSeconds: 15 + nodeSelector: {} + resources: {} + tolerations: {} + env: [] + logLevel: {} +fluentbit: + inputTail: + Buffer_Chunk_Size: "" + Buffer_Max_Size: "" + Mem_Buf_Limit: "" + Multiline_Flush: "" + Skip_Long_Lines: "" + resources: {} + tolerations: + - key: node-role.kubernetes.io/control-plane + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/etcd + value: "true" + effect: NoExecute + filterKubernetes: + Merge_Log: "" + Merge_Log_Key: "" + Merge_Log_Trim: "" + Merge_Parser: "" + +# -- Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: extra-manifest + # data: + # extra-data: "value" + +# DO NOT SET THIS UNLESS YOU KNOW WHAT YOU ARE DOING. +# Setting fields on this object can break rancher logging or cause unexpected behavior. It is intended to be used if you +# need to configure functionality not exposed by rancher logging. It is highly recommended you check the `app-readme.md` +# for the functionality you need before modifying this object. + +# this object will be merged with every logging CR created by this chart. Any fields that collide with fields from the +# settings above will be overridden. Any fields that collide with fields set in the files in `templates/loggings` will +# be ignored. diff --git a/index.yaml b/index.yaml index 3342cf391a..e15b9e4a63 100755 --- a/index.yaml +++ b/index.yaml @@ -17207,6 +17207,40 @@ entries: urls: - assets/rancher-logging/rancher-logging-105.0.0+up4.8.0.tgz version: 105.0.0+up4.8.0 + - annotations: + catalog.cattle.io/auto-install: rancher-logging-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Logging + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: logging.banzaicloud.io.clusterflow/v1beta1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-logging + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: logging + catalog.cattle.io/upstream-version: 4.8.0 + apiVersion: v2 + appVersion: 4.8.0 + created: "2025-02-25T21:42:16.290907782-03:00" + description: Logging operator for Kubernetes based on Fluentd and Fluentbit. + digest: 9b219e92bbc62100799e7e14980de4ed486385b72550a4b7cea8b5f2d4b003ea + home: https://kube-logging.github.io + icon: file://assets/logos/rancher-logging.svg + keywords: + - logging + - fluentd + - fluentbit + kubeVersion: '>=1.26.0-0' + name: rancher-logging + sources: + - https://github.com/kube-logging/logging-operator + - https://github.com/kube-logging/helm-charts/tree/main/charts/logging-operator + type: application + urls: + - assets/rancher-logging/rancher-logging-104.2.0+up4.8.0.tgz + version: 104.2.0+up4.8.0 - annotations: catalog.cattle.io/auto-install: rancher-logging-crd=match catalog.cattle.io/certified: rancher diff --git a/release.yaml b/release.yaml index 8cdd2c58b2..3e00b835a6 100644 --- a/release.yaml +++ b/release.yaml @@ -1,2 +1,4 @@ rancher-webhook: - 104.0.7+up0.5.7 +rancher-logging: + - 104.2.0+up4.8.0 From c36a909fa7bb797b063902238751c2ed0d28f115 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:43:21 -0300 Subject: [PATCH 04/19] fp: rancher-logging-crd 104.2.0+up4.8.0 --- .../rancher-logging-crd-104.2.0+up4.8.0.tgz | Bin 0 -> 170042 bytes .../104.2.0+up4.8.0/Chart.yaml | 10 + .../104.2.0+up4.8.0/README.md | 2 + ...xtensions.banzaicloud.io_eventtailers.yaml | 2467 ++ ...extensions.banzaicloud.io_hosttailers.yaml | 2651 ++ .../logging.banzaicloud.io_clusterflows.yaml | 2166 ++ ...logging.banzaicloud.io_clusteroutputs.yaml | 14812 +++++++++++ .../logging.banzaicloud.io_flows.yaml | 2150 ++ ...ogging.banzaicloud.io_fluentbitagents.yaml | 2535 ++ ...logging.banzaicloud.io_fluentdconfigs.yaml | 3240 +++ .../logging.banzaicloud.io_loggingroutes.yaml | 106 + .../logging.banzaicloud.io_loggings.yaml | 20601 ++++++++++++++++ .../logging.banzaicloud.io_nodeagents.yaml | 4788 ++++ .../logging.banzaicloud.io_outputs.yaml | 14078 +++++++++++ ...g.banzaicloud.io_syslogngclusterflows.yaml | 360 + ...banzaicloud.io_syslogngclusteroutputs.yaml | 2911 +++ ...ogging.banzaicloud.io_syslogngconfigs.yaml | 8104 ++++++ .../logging.banzaicloud.io_syslogngflows.yaml | 364 + ...ogging.banzaicloud.io_syslogngoutputs.yaml | 2905 +++ index.yaml | 14 + release.yaml | 2 + 21 files changed, 84266 insertions(+) create mode 100644 assets/rancher-logging-crd/rancher-logging-crd-104.2.0+up4.8.0.tgz create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/Chart.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/README.md create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_eventtailers.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_hosttailers.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusterflows.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusteroutputs.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_flows.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentbitagents.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentdconfigs.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggingroutes.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggings.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_nodeagents.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_outputs.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusterflows.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusteroutputs.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngconfigs.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngflows.yaml create mode 100644 charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngoutputs.yaml diff --git a/assets/rancher-logging-crd/rancher-logging-crd-104.2.0+up4.8.0.tgz b/assets/rancher-logging-crd/rancher-logging-crd-104.2.0+up4.8.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ff3010dcf35511de61a5171c22b0fdc4b653034e GIT binary patch literal 170042 zcmb5VV~{9Ix31f^ZQIst+qT_3+qP}no^9Kr)D}Q7} zW!C$SXN<&;gg^oK=lZ4upf(U!WH1t!VUu>}WHn?~WiVD^{jI6Q$tI_&$|j>`Wno}r z?5?a}%Oh@LWesrArR8|QR?m@h@`_Sfbsxv6mE-2H!E>LG!JK|5eRP~5;ec1~sgAIg zPzwe~2QdEhasPhxO+=}YKnetlP;b)qPKp_6<;TA>w>2w`<~VC6IAy9mJq@Mf^JF^R z=xskhbt{3y!NKyYqSdsz%TEeR!1W`+jzwNB z=AN;)HH$IvKz>p<(*ABA3zC*J&NPwCQj=q%(MN-9@l>F{4CU6hWCm(J@dsyIZ7o6d ziI?rf1oBSh3B3aQWnx>5q@s7R(oap}dqk^%nSEHW>6sK3D9yj&>1YGfO7Wao^?pYH zyZW%g>h?|pvShZid8=nf!YCbqtif?~yYqY5SgH8h(EZj5qphoKPi6Q_^QJ(n*er5M z{b#OTx37`+J#3Gu^j}~5A8SI_#yA%TzR)VPX`uIOXEk=#0?oPBy3%PMqi%%CO5M9u zp`hZHnCN*^aiey3-{`sDmq$C@>&M?S-%qbQ-9%qK@EpH>ZR``M-bYV$e{}p#x4XSt zBTYT3xR5cbEZ+`Ph^E`sN^=$L0H`CwCeob0`-cBLl zDsG4A`ZY3DuZzPs#e|2?CO#zkGb zZ}am=%*N{3k``-f@JYA>A11Xyl%eZ!p@D`u*kWi5dk$mc)8O@T3uihdx65rzn8%FX zQ+f!8YFUnPLK*h%aj0Xb=cV@Xgr=gYT#K{oEd!73$@!p0V;P#&h&8n#8o!;im%FuX z%@toY(O1BL-ACPe0_#c6MT8>&Uhn58{>vf&x)Ys(HcW{LN_+ir4tx~i%Ne1vEZ&7(TX zEI)p6d-}xJFFB9rBhZj>^U(9}L&S3#K(n<$pKu~pMJ|oUUzOtLe47Xgg(H%pDR6kQ z^7=|smz<#4j{^U(OudpTvFTYwgHIa6*cR1LDbJDzjqmk}%10cX`9DqWc#$`!7lvfr zpnW#H{Q1VGy9-Vk#b8q^?y@w zA`G8CpGO>;`>3VE+t?#+%f{+To=+V%L9Oxu=%@yLR*VbG(H z?ha&#oP>u#l{tj}yjpM)e7JM7)hF|*rvo{lJLt32(!Yd?lJPH6h+2NO@JhlP`Fo@D zsZ;lMfn=Ir!dde7_zSd^`Vso$%%Q-ha$2@qN90y#C4kx*d?)`S!Sd%KdoF^&Kr~ zzxaMU`QzKW;Y$?H?fW(5%Rz=p3yb)$G|z~!-YnK+odXVo5`Fx19G!qx4W87gXUasl zxDOh>pz3_3UGYt@f)X9hJ_`9v5LLR7EgoGj1pR8u z=-H^DC2aV{w$FlXFQ8)uvL&UGyO={WC0TQc{6PT1qEv13Y*id4aaj|UuS}E+2V}6^ zT>pq(akEox+-C&z;dkbNX$ki0y4Oyz)mC8fR`gRt!X-LM+OsX#gSYfXZ>%=kO3xy; zg!4N9rJ(^JQMmTfSu2QB3hOkk0I^_giGP8-ii`z)_X8TV;v7!l?B_q`lxtNH$P|syyxS&*da0e78gng{Voqyv6)RtI*G;ZlTuV+ z6l%*Nm)63>E^>??R094Jc5F;)YTF&(&hj`+XXo<uO|<5WtYZr!?VO^1&E+!}fl(i$pS9fP`zp!7XMYXa#*ArD+4gOrt}!4E)O?wAUTzu^)&1=qHl) z^s@##VNEZGxH#V4VMj2Y=po^<;*vLqYn0HrPKkUxq>8m|_Ytt><_t4xQs?b3TGqMY zi`YxN+UDM=r{TuxC>v9?caXwmX3|t1^9?!Ejd*x01bOu%PQW4Pw1~BW$e}>SE5|m}JerPLi3h*1oGOC2!S-gV z4EI#)K>!_Ch&v(RWd;W}w@7!`x#oMQS`}7IqQ5OZHcgIUp}@jct3c7388HJs#>fcZ zD1QY|5;Vz%GFOMGrc7@{8H^G*Pd+&xEG=1cS>$;1-B}AKj@`$fR0Xmoo5RlS?TM;kDA zil+qogWdF#?Qo>g~WZf4u8c_~BGBS}> zQqt->vS@Jmh9e0>qi}H4Qe-J()y4ItC0}<~JO9RGGdh{H69hB2y9%BScq)N$JYuhmO5>2+@ zR6&v@TBUvi2WjRI8Ka&X!C<{LWrJzM}}rDsBdd1QrWKtecxctNix4hFDB z_VbAErHYP{j1mC6MLEel{iZW;_|0UTmKOoDS#cjVkdKtD0FKhEfREBHfal-g2)g7v zdVU&WRnL?o#;g=S-9Sfa@-dJ{X&?aE-qE@E+1{5x+1{|d;o06iYuwP|jQKc;00cb0 ze0++`T>T#|0J73@ig6N)a_@>9rMclzBsGUg{{li`c{jM`mq)Z+V+bKIIJ7WE^4P(8 z?5YRQ^9L}ZYszG;){=p1cT}sx3?%O@dxMe_^3TF>nrO3N`D9x%IFfc!JEcjXnzqZf zZ#B)s<1tS$0eke>M=!w>^=zn!i;z27o?nMnvuFs7l1(KGSSuo=Hr;>V>mTJo?1TWA z!at|jrSer1^jn;FrU1tvm_88O_-ga5%kc1zf@PFE8y6eU1&#@-;Zb+SoL=?Kt-Plj z!g1JedMJ#Mwq%9d3e;x8%Tt`%%6sHhMDRPR@cu=SUf6Q{>^%;qJ%cZA^#VCNhrIh5 zU?}^?Eo_ZGbeV!`3*U{e=#>;I>iSz;0N+ttJbJ0ZMtGcwhC!3M?M2z-$^ZWG3n$3W z9Gtlw{L?3OTFQrB3HPUf4!k^xM8HlMqpu_rvHEyVqHpr_A)6F zVmxE(i&N813Jz$Q34Hjqq{lre`h5nrc)MKYL@b%5h6xKUR)$WNM=VXVgVwY@$`aDC zf>SaHftu#ER;D-Q{dnMS-p>)eZ*I@8sUKrt@AIVh{c-;1+imW1#+Uc|_%*k}_v7*P zUXJhcAT+o8`*Hu0@AaPVJ5I)V?fZO2?%S{BTbRiG>m&2~M%KCrM{MEu1fjlprkqEn zLM$c)*3{(`HYu(4FIfCaD>K2;3g{_x?OvGei~z{v8eq*S*McMfvMu7pM@t<_@BF1C z=o%=!qBt||0=f|Rr&fJoqVVW;KaPqQn@K}QY>;14muum0(4hjfJg)} z_f~t{k9+VY@L0@}VV3B&l*+QLu?*^mVgR>)#@$gYIN`L7qYoq30pqRLH-`{W<8k~i zzaYfT({3?30!J#XL3~khLGOE89pd;4@f=Hc;zUDm>=K)*BLXXOyc(|}Q!d>$=~Xn{ z_6wZyehND2^*9~Hqz4Arq+8(_+oYREWc$x#%Ti)2@qP4owV<8Jp%)x5XYRi%(>E?l zjy~@GG#v#qZ-9TA{GV=Kk!|u5$H&71JKnO3aY$ts5fe2yx4TCUWtT^i=JfGuAo3iA z6xL)Ng|%el(hCb%Wfy03>SP_#74(0{=8KnIP!-l!fbR)Ybx4L~jaQdmjPo^w(sv}3 zKWz43Ac@^ox0p3I9hXO%Lzqt1l03i!Mnt|8!*eP6BO4nGi;Md&x(WN-J*QzqIOM!- z5@yZ#-J+!kb)_?N0_!E<8TvC!Yqzf*XtPHpiM3jumP9*G$8GMrj>ggKyY^qDt}sla zK_Ld5)K;TPFf3ZIIn(9={Z9W|42h^q#NIjBmtP zc*&`3fv@UEU?o&DbRxGVq!9$ZVP*vY(2|)i%jZCDh~rIS(dZcaSPg{p#xlQs-^-Nw z=B&_2C*5dmoeVIABh1v^2O#}Np2KLl^zm-9DyHARjUBhey`>^VTu!nAvoitV7LeF` zlGoIU!oxuAxJrNcZoem(ijGgcIB{`Sda5SuFP$dwlC)L~Br`5qC^9y0KX~k$j`#Cr zm&Mvrp3x)4`>Od%O6CNKhFfKtBj>wB^66;t;V}|26rAA8#=Gm-4XJ6Y)D=BFH1DR2 z$OXr`wsrb*{?2xflwi+}yB7n^XGtn-$@yMVlFwAuZ}Bk1mOtn4_I%A5o2%Zmc{y^4 ztxCxWn^eA-=uDiXFRhex(P-Im0hG8N4DNM?0QiAmGWBAwc%5tMgnt@VX>7E$8c}hG zqzFoM)}a{IU?qlzBVI)#f$5~47x70kcu+q?j+_ax+2FjM5Jx z)s-)BzZCGp@*@Z`n?S*_5y&)1y$?~2Clso2<7ik>?mDRxF-omRPzBmF zG37=@Ut^4B0HT_J(>N~KJRd998Mi|-6wsOoWHHptz64O$s!HGZw-@wfvp0mI)!wqo ziD=$3~bXl5- z2iTME_w$KTm1t}G{#VL8CJ zGF-}Z6Eb_w9i$W69#U#8{;KiHj`Be}BY}10$DThbl)9-B>~97Z_V2M6GPIlBp_-(7 zgw-9bC=R&t&|1_3$c5T=(t@V*f(4JRQQLHL7|>iYpERTOs+$W(EYJt%#B>@%mrIug zt?4t_PQmLF)%dYyeYINYPJ7l|3b80AkgAE9!ok!-SxYt8zrIs41!)1OJsDJwpdLec z%X8rD4IejmM7 zxh7I#g;(s>dgzPOb2hs0Wv?IH@V0X89atRZsr*}tzLVy+b}!5A_# zh{oYyL$(yATOP%XofwhKh)2y#L`_Ua%}zzlPDkyT9ZTV0;-eM`S%#pB*1RrF9X7KF zRt*=Xfoj&6-4du>seSab<%%2f4%urMa0P*=dKF!gG}}82KD%iSv2(^QhEyf%r=)E6UK0~jzR0BVZ>rp&IG%WfhQ*MNPChl;2a?FP`_yjC0+vR z+h9@kCHkdSwdVw4qFzyUdtefO`DdA7{5;HDBH8%R4V<{rU9r8LvA$+F3U2v=XNoQ|1RYj(0w8{bZShio6pw$oohmGiIm{8HB59!W=sIDk;g2h z&v|N&UDyWrse9Dh5e8WTJSGxfte0zd�{D!?lo%{i-`sy*}4KGlfUrB{Bq)oy}k> z5_GI+WmT|H%57Om5 zJq(XHEI8Jb8&y1HZuW_lUl(xOz*b~r$mak<4mnO#AQCU4j@!b!L)VHDf%Mn8^BSo) zg~_c)nPw1H!%Q2SgnM1FS08m1est_eA}J(|9~0yt@UR(UhH!7vrE5_WYa00zGZ%Mn z6#tE@T}*uF$)tUZI}yq$_3V;olIv3J0oOZQ4U$*chhKlL6+NG|&*zU1ulF83ugBLF zdOe@JyNMp}$H$4(6FslbyTtn+n0T+-^`P#`cNO2ewzk#~4YL`w z#wR+8I{e*Cx$Nv`3y`*&f~XA^eayOHJM zx7|r7e%q;t)<0N#nZ^@xsrri_3b_`oiRy`Cr`kYo*v81?pwr8sNCqpp~WB*qFsIEeR8P+af0jwbc{7_i?6k#A#N;{vem-3?4uTnh@Sa=GOaIBcH{2eb*u04i zJdRs2)i5kyedOS7S?l}foFpHwcTY; zFT)_IaOFD`kOL5Kpn^fVR-r@lEZ*NF`#JL$=jy)6g zc_?Ej%wWCQcQyDP0SF_fyF0}QBjHcMtGHF&@+X}8CmaJN>_ycYolIYUhmCiKmcO8Z zbO;=G;FUH}m{*U3X9xUl$1cQQ?W`5~6hKXl<@e?h*!IOBYs#ah>j)cv&TvL27ru>P zr{wXIbO$Z3gCDIaU6PvTZ?XycJHstnINvKNJ57FN@4)NopEYCDQuka>Zc93R;3ps? zj~pF=7mi@x`TvOtgDo#_!(YJ!kB~HDLo@N&fh4R@J^`w8u921WSO>{LcE&j;4zT@`9~j z`2ON^Km+^=rR8LSd4Zvf5Y??TOUax7W>f!xktcmu2ID(we}OdHIoKieet}Nym*wf^L_5ZV>cG3w{0uv=d%9 zHlyJ;*_f+!`2D4Gw7fdtF@fk=P8H?ceHQyD`d}g*@J?ju;qV&ms2tri6gCXHD+C=D zA+OUE(v_UuZYx7kBv3}dY&m)6Yo;__(`uQLw4JtGhI^FQNVIOiFyeyl2S>{WHFcept>^nyZS+OQiOEbXi~(5 zZ=7O$7k}Akj)Vf&K7%|JQ!$Ye)t%Czx1@;f&uUYQFN0$PQ4^g$`Ckk13w-SL0f;j< zMazj*x96Q8E6gM6Ouq~cjmNA!@Nx2|?V9^D z@hqi~+s_;;T&tn#9mxt@lKp2cyy_Ge4X77fbeC0{XZ^MrsMU(2%OXC1lNcf)_MI#3 zq>g)je*TS)K|ClVE1>s_#XY8Uli9Sh^xq ziXS5=6Cpl0^)Z#x(iD{ngwdmx&Xvv_GeFIA@k4N+-4X+HMpr<6+i;mONvc=w@xrBN zt_H-E(NrNaP5+h4=E~-nOqBOUe{d9!Kr}|XyH{hl-R@s3i{H6TL=bzk{4bBy%(x-R zK`3bPZ33g5ob@9JrrHCBps1Yh2^bC|Z$u%sOiEg9eT_%j2p-8F9K|9P-M%Mcyu-o+ zC*G-G#k%t*?uPN)ZT@d2+fML5m~0IjJ~gW$i(=(c=1}+zM{)WP#k5sacbG~ZF?h5> zkwzuc)bJFmZkg#RiX~dHc7s0X50G7>k)1Q0i_A3F_*HgOctBvsH@LKAB}l>}ip?xg z6V*UhU*-G>sKC$?z<(8^J`id@*P8#h3NS{+z)J1+@C6pP?Fw+}J^zjyx6NnK@G~nu_+Xq=|sk4U>xaD1 zU|}g`8U=d&JrgwFlLfIK1C@;0_Xwr2eSflSvmk@v(9NKfjccbEt<(B zDb)cc8P&u7Jn-Vzzx*C}Pe0+ruko9<|KKDp<^6O+Cm9KHk|h}-0XRLe`uI3KH9`M0 z0LWdQ9*ys5C`N)Dq=Eb%E?!>abGAU|8-VQ$JyIN`6MV)Krx|a#&$fv6<@;pa_+ACCd-Ds2B<%yW(5HVFdzOS zjc&VAt7iWo+0}}9c+CG~vP#?Gye-XNUx?ZXJJa_++1x-PFKnPRo=vU(9ip5e4eRU` zhhD_~D4)aZqtC*vMc^Q-yx$&jsu|}*Nm~*I*P)_N^oQ2Vd(Rr+JLWuZ`+sSehj0>h zV4oQZ5~OT6k#_?0+41w_=XdfR7&M!H^0%*fN(?dzevI%|x~m0Wc@Yz0dEGe#*L*OE znzN{>ThY@0CU8k=NIC&l@1T^8@Z9>s`F+KZ(#O(WVrpMhXY!IQGJHn<7=j#HCc6`IHTAXqrZPrxX+|twtAT4+ZhT*OZ9`p7925hENla?_T8S}F@1oKh4m4>76qnA0HfYdP;GcqW z+$$hwb(C5f9@U>TuFxxoz^MIR9YNbHHjR1_NB=?<^zRp-ZSVrIsWq0w!;a>`*Qa(- zib?=S1lOHgT2hakd$894w|SFYV(5?%d8Z|2^k@Y9yjh09g~EyuuM7G9B-Z|)Z+nM+ zZNbrkTvxr<zOilo1Se%Pxc#p%7`6tSi z%r~T`glG^4lf{`l)6N6p?m-r5pA0}D`q>^d zy}puol0@8Uy@$a7>^d;d9*A9uhUdfzM1MW2UsLtw%MoAlz|%C8>7k39_X}?9)8``?+$D!55^QuEv2~Qzg>QI zkUYF+kAM%%+j^|&x8uo5oUTl!hXi5Rd$xZeJR4PvMQ^PDoxQSz|5BkGtGY|ICrnfd zh|S!eH=l!~OqEC*!G{5mw;LM);Hii|X zBuzD}>4LBkn}4+Z$*&5XSAS}b^o7}arahcUraVxMEPn_>RDBT5NgGuC%~?1^jw=Ua zVzszEK}|ZfhCVBwj)|g^;fO#Iz8?;WT&=3-ep1+FK}uw29Kn9Vd@4a z=ql4+I65O^Rd~7HL#>rPUsru5dR5r9y6`O?VR?@+^vzay>K4E)(8ACgU5t%uQ13~F z6@VIjr;Qvu)gm`xBn#GJBNTbUt94?hLHP#%@@Dp(JCgi9D9LRCDSSqIi!Vf^ z`aN_-VUbLsoEYWkq#g$TlBFQJ47jbBCQ{9b@*y5~A(--om1%^Lek`g3Eb-bP8;YZ^ zh*=@;3r1^U^k>ak1qo#g{VHB#+=$+Y8|9qZ2Vw1&TO&ijM8u5t9OAG{cO?|%R-HuC z0p!96{GD0{>EG;gf#S)30%9e|B5-g+ zmh+pU2X=%HsM_}$Wa^ZBgi<@Q(`c-#K>3qc5~_1XxV21v$RuMuK0(72DfOP#{_9Dr zz8-Y3e6qbl5{4bENJiz`Q!{zrIo+uczXYx2J8W*T|I)Km1qpAzJyMEr1+b~MiHVzW z9CU=ln2eT0(|6?WB~o-<-ooNc0FC7kkB)2uLpSe!g36}u7F|T<>!yTSY1hTuts5Fl zN3sO^b_w#MxI&6PYny~}tKy`_XM5efZ5!nxP;EW9kxI3^@880+Qv`C=na|H4)qEt6 zTlMd3$um0F_iQT>!hM?07ht5O`IOhf^4R$vvhRzqj|P>ZvjXB)&*Zz0ooXH)%k-Jo z_v674i%62^g9#SXRRwGrZ?!JZE84v}@aYssEGdx3s?3jft^xLN$T=b|K=k{kU!pY+ z+8U&>d^8H%j-lZ$y>pfoja`ad22O8(vCFv6YzUfEFR!om@MLc3m{jfwjygo}VO~|( zhre8jr8Y0c7}rDtWcXXJNqtRH8kejHRjSaZlSZS!in!26phWJ^X5w*V?2I)m8hC0iw_p)XM%3wf_TDe z!M~IMYiycf@c!}=^PA1ERjzVElhrMSv$cd!Yw=$vR$m*$0ze$+)0Exybg3ll^c>j= zA5M}EVq7^k@0&m>R1j+V4Y2dTAXpIkdxj)~S-Kz;)Yp?kkrts-#^{l}-f;;_R!lym zDFcg77j4bQ0=BR-CM-o+LNvyep&v`Nif~;^+$iRWzucaFkhOO`?E>OB(kh}UMBxK{ zlfn`f@u-+5ijrw-7yl%)O^$~5psByTZBMV* za#Ii_w(j_GEgx>HS)E7k_bDwk4Ie5xK=ZMY=E|E%IEMloj;5H}D>&Xlhiv-Sk(=xz z4Dn98tuOW=q6OR~=!b@0cYfYrv*0=4tju$jhDTanc^qv@GpGYBE)d z?pLhwW=-AGp>OWd#hN@42}g-sT|I^kK`Wa5GA9is{jO&%Z#plQT?@>fZ>eg;9P$*_oHrxMrhWGoar;RK zo!ceutz-yz^F9WdylYoY0tzO!J13MTJszEyyoy!3P?)t`PT~n#X+~gRE~S2J-RP%u za_ns8YL}6=PBB+y&&R&Uh#W+1PGt={RH*wW-a&ofq{rPcD-}7-7 zDW~i6+EUZ)^LqCYS)=FU@bzyU^;zP@2QOM4W6G)Av{gKF`wdH5YM%yjY&VN{yeUUkX~F69EnG)dkIR_I-UH!BEl&f! zHPC3$iLlo-R9RFl z-_F`<{-(D5qZ2^Ma!s?Rw)QJ=jnkO+E($e;emAq2;w@F7UI}I`QrE5Pudm+MX3gXI z6yK$$lPlPJTNzm%GpjY~gD}4d*oY?2jVSB94Xk_J=UH{&5?}Lw6)(RJ(HzTfZReF( z#MRJme|`;xR`J27!wR|217q1gGZpxQKkHqv74f;ySz?w2fIuAuoRs8g9ORi7JH#m=%;I5k2PamO~l z*0u)Pu0?xsZCuJ2P6a(VjJaQK8Ei%HKf?PD-Sh#{?_&HyKMRuoCs^U&Q4&jw0O($x&) zc4z%{$db8=iC7b|vHjQII@9SiyP-X?iWHT{?&!RnM7Ml^B?2d^Isub^Hn(n}S#A0{ z-D*Q&lP!uqWvOi)L`E4^ec7IXHwn(wgc84jEfNA6wuCH1`l&?%{vd0TB54vMC2Yo4 zQ9)I5VJbNW;EkU{m7{SK4w;m}?t3%f)M2yAo*;Qyzpj|~4gtNa z5HGB5^M(Z@auLx!+81CyBWwh!n9z)MhP`1dWY=e)U8h~J069*(n(4$ruWIVkha}a# zYMeN8F@xn^G?owdg49U&4&)S9wDKP3I7d-COI_XXcp-BWV=y#^J0RVEKX42W!<)fi zh;)<*-K!wI0~5l6vjg&owooPwbHI|w(~fX&kT1H%iv zfmn9can|~AKZ7*<*ToWE7?H~^2_;d!u&8)u(LAh(1fXYB2_B0q&d|$mP%%=-micUT zVXg?0`C%ssF+>q7xo0=$q})bvkxTzVB?VpK)F}`Lb-%`8y~u#I*@z|9ByU-}^8QR4 zMR!-kiMc_tE^002%fIiT8j_{0Q*GJ|eG@7rwJMzE(GuoQL!Ir1cULGydTStv%PzF* zGIA;avjvh4guX54l+&7#)mc1xceU)s*a2M7oa3+}St2;HLs9X;dZq|gn8tlm#|P_E zVzRWIM_w;`Droj50;A5L{@@xK0VPiYLIp7a4YRny_>g9U-hxHqmiPPan0=E_J0|JfLmIg$TRDE!uWu6^pfdXrB}l=Dki*s&LK3e z5JQM`11ewNKKxubMQah-64O?sP6J<}MHGlnGpL^hkhmbwDC%G{?WDOv8x^N73zcWD z4&^Yb4n<6iw!dHlhjqO625Wda>8bQo}PaeSa^(7yHSjCj!co;2|;1L)k2RIB%?-3PcWSD3$ zP5?m^rf(mSEE_k3VQv*+FpN|P1@H+n&oZkCJ* z3<<9?9ClGymVu5R`2-*s+_XP{NdYkZwlA&M0Jy)(-xCYNloW0%OsK`5gXOP*6CVHU zF}T?5I)KMrh?P*ys~5I0RLXn>3t`yd&7j2Iv7?1Z_AC5T|Y( zKXd2;ac;`N;U3@;WW;AUjf*rl*WQa*ekl>N{ULX4#5K2D4dz>SE+ZyqB1dOY(%!-2 z^QehcOwOa1%4|mWUv=rf7~6gk*SjhnT?@++GHB5!AQ3S=iDT*e6#-(2W0ClpNXzO? znuZ)dipZb&doQ+rbg%mNJA4?5aXMWP5UtQNY|wwn6EYpp>m~$4e~264#=oNTs2V@* zwBBBw3saCmGdR?y-KW%Q_ab_LX2QGK0xr!~_NFlo8-s}=2pGHQD|2u$3EfbF#{$=b z#?o`+Ca_|J<5YPC))h8@4`}T1?>!O}z^lj3hVkH0#&84G?L%;Z{JzuJhhPUFj!?ff zjv$eT`AtWEr^mW8-~#@zbFK@m+G{N&1pIBoj}~_9<1-2N0Mw36G;eUf27_U>ty9)i z-aDO-=}hS9dUr%JbF)DeR}@vy0abZC^l8#8o3W2%rRhW4mxqZK%+Tcage(Bhh@J`} zODe9npS2Z={IxpWL+kH`_nU9&Ui??pJK}?`@I*{QGSzER?76YlWcsuKTKPjOb*WAn z>!P&K70)uifv4sQU{&(_OJsTq*5}vZo4*5ylGOfDJGUQj_y<& zXkWC9K!*C9DQjYhbI-5@`?C5Hrzj}1EFYYiUw|<(V4DqD*wrbou}&H-VoPFHRAvSk zq+0luj+TMcqTWu2;c!YhCW!@%Yw7TEw2XcDAK!YJ;-HLM0U=zI3JSu!TFH~K^_d!l z5=M;_SHPp-gcN(oc6d(mfnO$|=$qRlqQBIJ$wb?_=UHjDHIvI2EvlW~SglFqno0zx z5bDHHs0y)k_;neenUi;8=6}7++aK!;hSOEIM!KNBQ>ELJ8HW)31dpWEhmjdrYVDAk zQ&4nfoQaAW%NfLFY(By6wCRbm7!x_X*oMLMkg!#8kFU?+=y$5U)!+6^*PG8&?1#-ddZ*@y79Sx3?zsVMZ z!u7b+9(bgmFD~>i`d^C&ehSQ(XFJNe6R%YoT1cjv>i#|J_(azFi);z<#P9k(zT;$H zBTK65H!SWHs*lFv-5`X4dxi%astZ@@ z9lcwd8I9pSg9P)TPt)MKkQh>oNKK=6!HA|EB3rd2e4#VAXxV{v8g22k_C4@5wB5A! zYLv>M+C5%r^A@ONq;hWAIbI^2c9feD9>QDh{BANqI{nr2{Y*14-Q3)k9pzIftW-5_ z_$G}!)dAJ-lIV`%hfg_Q%2%*%s!-NlAv^ z^x}gHDfq~`=2;aXeY4({&cj#T&;+FVQ@viJA>5L+O>VhBcemT~J34;z7q@Uq<#A8o z_7oOIUcMBa)M6m2dUK`&$Djx+iaK6~gTXUfKveS{G->2sEBtcvZoNYEScfFIZ#1xE zk(`LpvAftIv}+RuUC}JA>x#72v@q7S7xcjw?M~?O{T8Jl(|J8|oJlz>WQ~bHelqS7 zNf>68OTjERx`pW)jf9)!f7)khXVsSe%wzL){wbD*H0U$$v^VQrQt^pPnFQ9xPN1dL z?P|Q}D#lq?|MPmrC==8G1+Tjjk#?!W-?wB)M!M+vL5k5Z4addxZ6HQlW45iV>IG6F zg<;n%@~bT?piWNj-F6p~Y{0y9ys~2urEl%W8v?b?W4n|6Uv(QwC z9##AqtAyfYWkNOOx3!z-uB(m)wP>JyvQ{R6FMYbiJXzmT)?7R#X01TmhlW`KZAR|gVcHmw@54Ju(13NO z19_Dg=^I%e%gJVmbku7+6#i@O0Btf7T+7$*$_0JijHEGE#cgMOK&2zJkrMJ+1eH5` zHb;r$OIJw!TveNoQaQ;R*62+%KUB~;Qs^44of^k}iVk6gV&N{)7q;1@=0>T$L|Cj$wc^ehd zE{gFW718UCOH5M)4^>>sw4U>X5?ZST!99dO6 zfX1cc$Y4(J-AGc+eb zoAX1}GzP7G2+^0o@Qh6$=}KU+ki2P~XW_tfgk&~D@q=2ggIeJ~%^`JWCJnI7v55@} zZ5a)(lZ1_8I?A!+I9}exW|f9@zR<&}Yb4P(eJk*=?Ehvj8-&%ls)riL$mj~VDiaT| z8jKU}&u_}tg{?sOC-6|9poIN;xm(5f@R!01!tpJ(L~!Wz83^qPNRHw=L=|3k3wS!B zH71-`b7hySfxSgnKH&QgXB*qWjLj3j`>Xyo65EHI&0`t9T#G1xh;hI;A24B6pkCb7 zdeTj!^?X?SQ;Zc0H{p3W)_gR6-WN15bZ-{wukV5(Bu4*~aJO%=d5bX6(@M6Q> zGQ{u=AJGV6`hD^MuI_Gy2t~^$`6r`sJN?}8b!p1*32t!s7hlS2@;KPg2t!~37z`oM zVi`^^7=prwLkL7^Fbg1R;u+&^^kuB_ z$Q%Of0W&zjDYzMs(UQWUMx)?|LEx|vY+yGRdKswg%|&tAVF&XC;0x~o0@(OCnCT!q zt<>3r@&xtM{gPTgVWST>0%fj1uMDUjhkhA&Bft z5VC#YGGvFfx{ex-40ZsU`hsXuaUs)BA4H4}N6ut~&n#djmLaECFjE1NvFVv<4^NL$ zj+k9WjctTaZHA9+Va9hQQu&fQK>4qE#3v!d*NM#o0F%JZ^3M~zQX4tE8L~06)X)bg z!e0~)A3YDjp`;w3{~FIX0!M7~@n7?~4TXxw^u0}>(cTqj_;NVhTDO#Yo1?5CB2c09@EV7mpbr$LWR>N9*RbcN$mO zCw^#tgdK&O&4V!w1rAjMj_BR%5hQwXD{g;?7=|h~ODLW=gCy1ph@mhTL9RsNZz%5Q z9{*kbir4%r?_2j-vlhq(8w=~j6}X>n(ud+M2H6;O_43?(PJ4cXtTx3?AIwg1fuBJHg!{ zSa6p>zPb12-skb1Ie&m~i%+dO)j(Za1Q4CHhj_1E&@#$0Ic z@b)%xM_Q3r)Z=r(Z`Y&lA75aG?xfzzb0pWTx8m}iNOj!5RjZAc0`#ahOlDjW8qG174bV z`?(xxm=<|X+_tlNDLey;efbQ{6)yP}&|d`wuTV7PUO263)0^XWG2hHyl{p@-dy4Oljj?=b7nkCyOV+Zp zO%>SkDdm36w-;O)cUW3NnPSZ_5#&&%We0A{KV$=HGrK9QYho~LI1ij#xcawP$`!bw z0J(9S0$AVo_0HsNPpoRLCmWVOcnVJO-*_(9H~iSy1TwcCplXGMS!8HXvi|xb7&uXS zvf{pX7QV0r58JW>?u1P>+J!RB=+4Svr;pJxeldxw-(_{Rsiri!A}ul4Sdwy3YyAE} z^vLvBGd)ApIaGhEZsE2-^bNOC$I$eSX;F+RV8Ozh6xT4kbt6@-uHCG`&c0O+}vYtqeBnWZFjlKP^Zc-kFB<){!Z0D>fFBQ76SoLOb`|q=fa?LYn z)aV-h3{Ed@wNn|einDgtQe&+t^8X?L;~S@GMzGVJx7+^^0Q#i^l}G=^WSiftG6u|i z{wduj5SxE9fcK7GAOWa}ZDZT~F3$myU4B|kOoVSMJMZw1q&{^;r29|d^m`DdOPRe( zl=}Z^2SU=^CUWcJIuVQb^91B>my*bnT|Kp0$^!wB!0(^h1bF6dru5xM39ad{K}kPJ z$T@ZJG}h27lmJB$*=)MdNk4@0LnLf`!{RtNVGEFQn;v*}S4by41_97+DowEZnebR_ z@o{NLDj9eDf_5S!HwMqS+#sZ@d`l+EX;)rX_EW0yzv`_4>(B`s?T~DmH9=(H+V{|% zgFTyBt>HZC>*7|=qfRFIO6qAPe6Z3UdXVM@Wfj@UKQOpa9~`OMPgxb;;QVz=C<7yheU@=AnYSj~}7l+14g3dM-eV+i#TbZgV zI7GCr65Xvz93KTS0HiI-F!M@9^?y`5?8^L<+;#To6e9+Db_Dtiy?}2obM0xBgPM=$ zW}sig@G2oI9h}>0QoyZw&Y0 z#sqF^hsLb^#%Q>aZ4^Ib3>_s_NsUlznUET|SdlihXyI+$vzkT_HQHuZJp^v) zgx+D-HZmL_am~OE+`t5&ZXaNJ(*{=NWrrNuPzI`!j`6)*vAS%EoEKE##fH~|pL9D*7`>2#dHM=6t@Vujt?_E4c25(W;zi_;GOH>6Ty zlz4|)^y87T^$fL$%#x4nLbS^jN+Bans`EL3+XPD?gA5r-#F*2&1`BthhjacA7rYSw z;#J5)lkO{g-UL@VrBGx95XP?uzCsBPW1kv%q&ku2Gt7NV;$4Rt0szloO7d*%&xC=V zV_GhwlTMXXD%3-Qz>_%_gZ=kp0zJ#-OmSVkAOn7ac2v7?V%%S>Bs(S0WuzJ9*XhkT zGgofj0G2SS1M(S$3Tp8-NP4H;6$3hY_vmuj*oaJiFypreW?EY;31@MBk2TgIepTYG zK@~6RN;7BNlX3K~rQ-st&6JF}gyGa8Npi#ah!F$(=7v=w-;UkG@Vm-Z`SMv?tI_e7 z9=Ct>RF%bW@j*nY4ZaW_qb2C;MmZ8CL?{#9gOR%)(V@@6dPfqEV{0idDhN{;&E(UW z94*32^?B4=P(sRQyd`llU==w$+0TSjuKgc+sgBt$8AxaN^pukpJr^Rxdl=Z06;joU z{_<8bExEmEzS5zBoassb}XU@HRN`JQE^^3TUGqwTnOSFJAAXROP~ zt5G!T2^|-37iXcD!^XJPx|ukQTl!RPoKZfyFmt?0-EU~^gB7ReU8^bQH7rpZDrAf5 z+hH*}n#Ou@_GB+j7?2@GBSU-wBjJ}a0W1=(VZoDg+?aVU?DJ^)Jw!r&svg}Yw<;Jf zDHaJi=M0&TB3Cu=2O^$7f^$tW>;($9vk4-K-lDDWD^Hj>lBmSy9~N)$5)V<$#mA{c z#&e)Xo=Q80nOPToeJb5H={-3_ofr%y#1*!@C;_!u098*-R~Vuu*lq1Y^8O9(Tw& zYnxgnU(R8EW3=9F+EB_@H$iN5;3lN&Y)|V#41`{#xYRU@E!P37;Fgv?_?WyeCG4vv z;1ISbt6)?2d+M_!w)Oc4Kg&)&u_RsZ?y6fmYuqUt$i-C{Ue|+GjW%zRI*bV5-0BJ+ z#kq*H9`X^(P|CzgTzMHu0+0;#WgC3p!uPYu63;IKs57W}aDC&?Ij8Os{G`0mrjhz@ z7Wk#FqN!iVsn_oMiZT?!L`D2;XpdsLrb>Y+xtjhflt;}BTk2#+NrU&^2*>-alS)1v z2@)ESIy~w-QT2-=6}=Zj=Y^2Yw>Y;HJ##5SEt7lu5(dm(fHSXpW#_^NPX+p_u>@hsu=#4)Jb+7zl ztGgARY)+a|&6?=QZ*dS_*m1)D_!XyoaO8b$OLe}V>QcPYBgdu6$m1(eK(4nWxh@@l zr}&Z}Ij%{0Jg-FwH)dO1(#1j4`Z}p4VD1GP` zH`kF(XSE2Q(N`o8%1#=Z2~SiEN@PLZ!2P}z*Hbl4f%-ef5L&!i_0oESt;^pP9)*>U zHxzn6bQGE9dY%QGKx~#pTw)ZNY<8Aqh?tc(6x+{e7M3xY9=BSgqmu%$k?k+Qk*f77 zkt9%d2;?oW7i8WH} z3_rYsXA3Y{fX-Jsx+1FgY$E*E9<$Vxw{;v(R(>0WQAbzLJ@2z;B&HK=Mlu;e7@0?Y z;bRSw#zL1$VtQBm6i}vY-MMy*Y+Z9#dk#>9Y+VFW;Lsiv{;@_k#n4~j0r90oPs7yl z#Olejz4nm=M+f$cdq4eDa%>buiuo@X;rn-I=^8ZKkXse7&yvF^>YTw>vMClgz*48Xv;HNS{oomtPL{?>`LdKWv;cHQ5` zTjymZ^?eKn)P#*wW-@l^V?}+OZ8Qb#^Y=Oe6y|wcEC5<1(NTKiUEpvbKb%v8wHf4l zc<-5vq;8GpR$?yVckC@d7lX9nSb?E|N9m5K(;=iyNrt);LrQ@x&N>!ZB~?{*=LHmw zhyD2ENtcIUl8l(2Y^akVvr6cfvFiLNnQLqbnI84euxS)}$9hLGu`(5kY8wq}R#lab z;hz`$`B-c^7i;nSdMiv)LcKDyYS-@$K{X{IX*LN2bs>;CECi?4r|pB%P;ym&=QMgb+J6m$S+#YYN?91M&=%2#DkX` zag`mO==Bqk6NuFYkEYR=Pwh8;d}$?&rt%Tp=@R!DdKP&dw>90|a)oZPY^E)plWP_} z>jWm-8lp{zYF8iPmvuJ0d`unX<}6`SP-I^?Iu92#u+f8X(5~*W$vXCf|9=r$`Ls*z zvtO}x5tUC=iSKMD-JqvLGoDe$L$tStJ9*+ITY*y*s8e!_LTJ&?@&hjNaf*q*?NI@AK(h=tt?9Y$i4OnP;j}1+ck*6oRoE(7l)pPSf-r%NV=qXq>)-^^!6&Qd zHziNd&Jq)-*B6b!Ri;$gSgiC7X@1ot8m!)~2;znH9p2cDH{Ol%>d1?_vAteyV|8$u z&i=T4NiI{z8m_~QgOsJYE2|ZDXoJ(vsG6XmL@0yN-vT2&6XPvB0@x^W9@|8&grk!# z>`6C2CsffqV`XZ_WkF70lxPrc{A}IDT?Q*h2^y8IownFxnP%ti<`I%Pdh=`F!g zj%(5HkP*>FC`g1e){)=7YXkMxsPs+7A@GuJ3$XhM>XdtD!YL;_-*wbWex9s7OUBVF z<4uFCWWgzlZKyYg)_3@kqQL9d);{Zskqrfc5n@!>1&Fhrn1y7lEi$IFp6V{eXbQfj zvmTqXo@5@$s2eg4Pt70_H~xwsgIn^2woH=k(irQC%>Y^4(Qd~P?k%Qfe`Nbzva=_x zOUa}Y5gSy=1$g>*(7pqlZISPn43jtHt8~RoZ-qu|uS2(}c@{xTq~8M_42fE&9=_G8 zgr$X4)bq?SP$rY{@Mru8?0kas!wQ{Ao8ltqkix==lyJe@LXV1=$xM>jR;yr{sT^Xy zJ3S^Z$>E!3{l?DbQqr%S7b(7gJoNal=O&GW<%Xfa{nW+Nkdz^qb5Rp(9#-%> zI;7$~n_11eq8EsohE0qHOYyfJ1nId&}HASL``fX=LZ9|h7Q za-t|U4*`I8@90x{(Ey=id5FL>8a&r=fE3#~3F8{GBef#z1tsUuCGotq4{^;J#6IpQ zer$xTQXGONx;?2=n6P6BGEFlEvQNjJbsZHA$3E`wPeU9iaX6D^JWeW@_HMd6o01Ym zOWwpEu7eA^it{%ACXzZq1+(JZmjQ6z%b#VS{O)I5+Sz?%${H4FT=bpD*j{d34iu=X zn7#ZbW?%WmWK^8MlBYf(yg8W9gulEy!aiAARz zYpeMv%3N~n^0Csp1}dprI@SS1xz|NJ@qhrC0*yklFfQf5ECOu{Y4B!2K>GUcun>o& zzx~{XtWWteD&hyd&!>(_ocQp4UGW-TNQv(5Utis@ekXhusZoHQgbmI%#K+^Ld%SM0 zMG@WXPOhq_Wnyol^#w}8$X<{?{5>3D7rjW#+<{NfaBc=tfoZ%*_N^In6eG{m=uOhU%r zt~Wk`YNU3>&FFhh|8T0jbS8UTUdnZ=B-j_3s@^i?_Et%R+?afQhw|@K(Hp{A`wdAX zi1>3KWqA;6tDlTm^c#fR74*dx3U@+X55aT18|Gpo*h)PZLC=hY+fQM{;1h@7O*k_L zy!~*(aI(v2KYV;33*V85QG;fgqWs1>7!P{`;xxqgjaAZ<=vLpM{ua3V%-}Q^n<*2~ zP}D*)j7dc6Cka|m>{<05iATy1c!TRMCvn0T}dNmv?Y@)nj+TuLyqGSk7aUfz}FR-OCnk~y#@=d zP@_yFkhNmoDOT#i;*Uu`@eAwMMHv5*%0(TFKl%PQuAmg;t{XalCA=R{`JnFp9n3h_ zeWKdPg7(9X&E1uDwfq}xq362KAJ2b!U%+<{Jyp1?TUO8LdiXg5mMF3eF0I@4ZIOI= zE*Fgpmdx!=A{%gkE_f|kM2k9$1;Sv$qR38S!eWX#%LyEJ!UA7hVl8?Q=^2;*u8lt{ zgWbpR zx9g;R3bp2j-5C}lgDtWC@0#>(qNNMU2!xJMC3+H?Y%Q(^? zYQvUi8eu9wPiQ;gl)u5Vk!r%twzJGg!#iZsuvgx~M)D+W!q;UT%O<8aKL1)E9jNN!|c%OX%2K7b&QJLLY2`I%RbIDV$Qc z^oV_^y=R&nGY^Sm0BSNmbtlwm5cv}(ayOqO`S=!yGOMyPyZ6^agXP8Pl={Jq6I~v) z)Z<-yPRLtz&U3ZXu7ordeQAG-^Xsph9wYIj>!2ov$Kgfcynl!@pwG+4&# z=oo5LDornF?})F)xaJXK&P%5DC$5>MQaq9-`g2Pv&VpLJEd?oN_z% zpYtQH^G#0bnvP#|t7#zV+~PfLC_GtelnQi-N=8#6-KC})T*nx2yH)`n(AAXZuU;;+ zEJwJVtUQ)WZGchCaf(g&OW@WLU18i|9?~_5KZYzR4fb%akoAf;9d!qopNJLj|Abj3J@P#{cCIcOp|9Z-(U4pP-7`mt^D7)!`O_u zAT>?C3xYJ?r3+;oi#@qJ(juAfDPba-@3{&qtA8IvS1-N;;XJxoNId{`wLnOa%#TZn9Y=%o$?Vi2(xbWazB3S_?{|b=zepO% z;)I~#ynfMKCypu-3VHp)@%lwJ8$T)(9?k!;2NimCCV8(+I5hJEJ{lrBtpc~}H+U47 znKs>}Ais;J)O&eGlA-U6q>Q3P`gDPQda-8xaoQ3kq&FuUqoohhFo7z%%_GYq0qF|4 znfqZTxG~8JIfQ|;{1^xD1=)%jIIgoiWI}>1D-Zyk^~q8-)Hlb6Uf+L3T95y*gXe7Z z3F6LD5leeBk+{miQGg$nCmEy&KS(eaIc+xfWV6fs5?Z(<-&ee05lGXDg4SQUZ6OuW z2&o5aA1X6A{cE!W!RfROWY80JitU#HG3qqiez-0%Fw64lFA>MmC;&Dk=zQG% za7{Q}UF~83+XsFX%28xvx7AeRM3RZ>Z20Sor=SPQ&8qB?ccrJeNZ-5<@~4?%-|J;+ zx&^CaD$D1aL{)qzY?5IVE@Tz_g#Bd*i%R*u7_<^C=Y@7qpfy$b{Ym8CBdRulVQw=ay*zn>NRU0tCDnSHVMR zxqa!hY}Ah@8`KrwHV$IJ#TFxEFXdFf( zrfc!zgQiQ8cK_|39h=H0U17Te`~>uW^;h38V;4#a_?@7$HHdh94ZyweRD-$w;%cRP zX9%hdF&S1wzcbzbkt0M>yGq(iUZ*&$1Duo`wQQDy_b2JS?I zO_TM?4lxZC+h9!8F+R291obz+^9PB1{NFe0ja`JP{`5I<>@Vv%BCq?$dYbxXY&l)i zKdS&`&}Ebx^YrM7vL*F66M#FzaM zN1nT;-;S=;hJ`;n9**ihd5nrc)g)*xH5Pm_<<&Jiad8Z1?QraY>M-zuBiwmdcirnk z3MX^lOXdN+L1rr{Cti^eo@pW@h+&7hnFa=AJMonN24vtisEzlnTW<=qA=p-E3K@Y? z#s4N{K-DKO*oq^uyCG`)KY^t+9xZCC>+FT)_89sO1d)Zf$WV zDz>JHm8lnOl*)uJfXt@CJfz^Uo4gi*9{u0h#D}h}!v4-_x_ht|DlhaEo@xaqbpS=N z|4Z*gb9i*&w|bL?h@LmXluij2DgAE@@(N^%dE?NatI=*}S}i=H`i-V-{pID+YoZMZ z4zhJhT8x(Mtd)BAfTg&qfc!Pnf0UY%87g6h^k(8a9=1CY6y#7cE+w{UyTk6JyY3ZQ zGQ;S~rgU9hj}0W&n&NrNAa}0hsp3Sm{K&~Rj2}Ph?k+Rna;1xnx2VYNrQ7JmRYhma zVODt32Gcyfh0G(y8#1o4kfl5!MjB`n<=<*)bZ5#t`MR=0NMnbs=fu;h?`z`@Q1f|F z?tZ*@)db>r3{PGML36OBW_Yzy)}+4vn{X)yZaI2ri@4{OsBs>yNh5WKbQde?j7Lbr zs;=KkIZ#fj%y54UzV5M~N|^@(jhjep4T$8)CR5RN{*Sf5KiNP2m43Mv;Oa?RQu5)> zZLob{I?;QlXH|4Vc!Orc1ipz?NZlvq4M)=`uwl7%Z30^qrSeO0xc=yfC$?xMO|PAV9yhEJHw4Z^McvSk-)la-lw9`YhKt`{(J`nl z2dCDo(4c*TH~GNWWg%P<`^6E79N^V$<}^b{=l`nI#f$=C^2jzPwiiAhNA!K^=eV)yKX zia5NjSg+<$n)tykeeA;6&hTvstZe_rE=DC?AJ?s3srGJV_*zYwgu)PcL1Oe#VV(Iz;f0HVixC z7Qj}aI?hYR0ebM#tn?yaRmKXw6FyOED>W`PK75^k=)>Ph#df$MbJ6qV_L zmWjQ&Sf(9dcRl{5J*go$LXg$=+_;@GWV6jvIB2u2O|3QU1FKhcL;%)jv#mAVc`a6- z>46KcaTgnV=2CGqo!X~S=(yKWS1F33ysPsjgnCjZ%dz_`B@(ghU=ql zB>BeUi>|isJtkZt!WgGxUQdOZt^TZB-lT)X%XdG`lsXHv=L`sc{jASoYqYbb($lQh zjeE`{0o(`ug%{Zrn~1FRWmQAqCPWA918jWr^_95sH2Dc1OiCljsT-#euV;mQHLic-{Fr9nNBxUv-&LdzX&SW*Klkx?5MXPZpvo(S+|~lJQTKayE=j@Tzh;Wo1n5W!HAi$#>GgDW!tx)^co}@A=W+ zzs4i17#~G%bxpM-H#AJ@y+O1-sz9XCesbH`Il@8wb}*yjXe%O18}x9g3eP0t$f@uU zqbaKZAx)Qz^Ev9r{3asz{oRL>%&MXRdTB1W1N@%r1!|?2G#v<-gMEJA??ni~iW&@+ z+s{hQa_*by3X<0wMe(LY>_vCGcnPMRp~=z7dhkT20VM1|b3o3dM~BY`{tLvcn7=}b zS<%Ah!lYUTnXxp3%+S3#Be>Fse=>y-Ay17>m2BzhGbw4>KN-q z*q42DL&&yp>q!r&At=AbLN5_E=fhzu%ImUM+mDG>w%{POr=dHd$I70_gN?`BITCv9 zl({XeE2u)XuwHabG0hv=o}xWYi*z>37w{kCz*(Et+|MUs`(X931ZzUkMMwSISlz0d zRa@T%^d&dxq^}ZND!i9|H7^omTpm!NH)mu)c zNrbM8yGQN8Nq?y{)>BWC2D^nvUdF9l&YqyclyT|2rzYW18cuN~42XAoSEre^A&9+? zAhoMpiHC$`1K?G{s(`E&J4omYw zO3gD5!nA!9jmWfZk-s~oG?yjX-uX6*Z7*xNmgDqIrY!$2vwz(snMVG^&hA6LQ!3RNnlV(fnFgf6HS3jNB<$6|SwxDWf!uMjVq$l?CJyh%X$ zsjwe|JwRT+X9~+JN3<@F-CU>SEf??pV&1>^Vgh%yefPt7`$8u{uvaW2#N8Hxfv`Wm zxA$5`ih34MNs|3+?Qj-d=%cS<(h0OJQcAAml1zLG(BR)>Gvtz!jv5^ z4x1!1rcrs$F52P%K+cXwWSqD5e2^7$>8pG$ zFCr0A*AOL+L~S#-7Z3T}3m>;taQIM?*(zft)<$e^y+P|F=r|D)a~f-VUw;~oxqH&c zNO(SBSL#~mB0Lx&5(Lm>i?(N_MkZ%JN!R);1Q;`+CZ7c{UCk65%@z)WkS584zk@9{B=d61W!mMn!w*ZP?b~38C25 zTE<0t(Ad%J_d&)*`MtqH^YOf4g6sCzLqBbg{r5j_$p3?I2g@=uFG1{Tsz}*_*7X`9 z)ezFDi}bP!Cio`EU|X%)>ZxUgU42>22*xrpDuY!FQ>3TD%SLGvVxc5aN-h(>Qkb2O z!Gd)0${HTit+hMu`l5AO;7@qws99~MV&=6rV0c z$#}#vsI$CPxX&xns-4aBH3*(^h;xB`R}9#)>{sItS8jo;d}%Ts-i3x2UNZ8l*dpG&HaJc#kWINn1+cEON{+zjiZ=dF zbk&b&Ekc9&75GDLrLZbtG>Bw8z*UuiuT70T&H}+G;cP?>CE=_Fd;o2Y&B3}O^;eh_G-}B&2flHRQ%J+cy^(%u0-C4`S}o!uMH*8YV~v4@0F&loRMUnGrCl3SqQJqez&90Bw{slx zlpp%4QC#~uh^Bggpx(BPjIisS-(6gks-y_QdX(;{(^Z#DX~5RWjryM7_fcqU!JKQ6 zw0$d#FNQ|`;o1BXufXHb_uv_){j@g{g7=ebr0+;0DjS07I6^{&Q@7JFnK08Lg-deZ zq9<=dZ6Tz;6O}akF8o~Jr29?s`*0J#c2nIPw{7P{QW0ksudKJcijZB|G^GUYxg=B7 zI8>;a4Z$}dY^xN>R_XoNU&!Un;(PYbDdO4`tCbBC3r%%|Q0vR|bFL*k)inxN8NkB2 zvF``xuDi{fu1(sG+V?k2{wseXKX=eV@8;j72X27tj@*q0c-P{i&hf{dpGF*r1}~4- za|dpxonH*5`sT6To9Zd5PTdF8z;2f|xO#g4{@0d7KjuS!d!JF(hD~YcElU%u7+K3R z2rRvL7IGkov4C!zj{vC3>c8RSWo9!|OW6P7-1ILKCwdbR`)+m}b7Ja&K!Zcmjx9JJ z$QgruOP8FsDXQr`b26Kf6B1N&Pvbb35?kZyTC8~@Lj)3Bj*Dy;j}XRJP9#`ogaDL#+;-wW#7Nc~)Im>Hqf zvEtEK9o^ptnuT7-K#{IU=sY#wmAW0S{}awd+aj@m>PeFtfthnn1N(Wi1T1ScQ}(|} zN+bUQLx%e)p`^=+s+qc`h2Sdxdb>k{Ekto~;8@Ry>8 z*hCqrw3%-CKmKaZScYs zI3Hhrz<=|JQXxduzrc9UQ$B6`p-YzH+V=pzB@1a-xoR(#+di#}43!thmhhYNz&;EF ztbOjg`}hVFgv|d2y=>RiXeQCUKsH$&WwGRFv2|;CegsiV+tl^Zr@?_*;PEN6XwP9k zr!r&oSXYDXeC|~I#8B2uJf4_o-j?o03&}F8B>$yNsqU(b!%qtE;feg2n`0#$$r+&8HA zh*rKAn9Q=BaFFTpT_OO@h^cIKAPr66+GP`>PMEn9v?R3-8gJ`ff+v;fCu|8c)tf>v z<-{o(tnCUiM*pxYRLPr zDb|f0d2kDBBeYto2Je1cIpC$yg)T-W;K{=&Ht?m_n7XR};)W@8zADwK=uA@f|3(0_ zFIb3eYWqHHc3UPD>Yd$78X;NmFAM|xDhLLoRG%fek(x5-#nyM@ufWMcjyL1eWuI;^%oHj$r#Zh`>RBS040C@_FX+F zPlWebn^nveL2qUiy!PfuW(0eb7Od2Xo;ieI=KS>r)zxdj1TiT#Kp#&OPw)(VLF@rd zFg5T&YY_D_*`;!VkX6|S5bz?q{*x0V&o>M^Fuk5c#3GS;P`q0I3}|<4C@oMui(JOA zU)dfj-i>wt-99_TMe~C#| zCgy@r-Vc-h8sLi6QIZbtBKFcYr`3T? z_1Hm!<7?SCN?Z*xufu8o-fDDEYVqkcZ{I_APb;>JN>^J|%*5o9*pQ>c!+3F&sOD`9i5{*j1=ze5$8q+nXT0HvP3=C> z!4iRYSsRW{AR=x~cCdpu-M_I33*JY}DZ6oWx(F}Q1Ft`?WUT(*npUZ!oh@lIk*RVL zm{ixkCa97JbzITKO)XIe2{3rE52r_W6_X2$vWIpR%fl-WPmWLKl=;8X^oaT*sRKX-zhZNsA`mW$#v3g4RdR3+A#5fJ zC=q%Xka0Uv}U2NX{}XC%wVq--{sxISW)u%wU=z^)a1uW3TXt)^dHV5eRidx zcG(~BfNxCyfH%Vt_-v34Tq1qBcIe=hDoc1^sEv)fwoLJUCHz)!NGnyLt&K=km7;8V`k$ULW8Ijs|_wRp}r2`C4;2f;9N^)kV9w6;A~ zqr74fpPZKt%p5`9>w)4PY~}K)*RV9h8Y(Oj_QJMB8ep3RjIxa7M;M{$*G>SlM1(uG zUhK$fz|q}6(#bX8(kQKc$v^lEVxsya&$e% zknt)nZDPK`ajn#TLfM~XiG2mLK@!dG=HN7eev>?pH5Z0PUuyi9Qvd!^p>=u@$j)p~ zwMkAx_HiiIz{g-O%xAcN&Eg@yH_-I2GQVsjli$E5WxJLRHWV)fH-_w&n&$@D_idu{ z3aR)XRlWtN%9CrTTH_WooLrVEhc8~}&%?x9tuucIg0K8lMvZ@oEi;07DX_LvE4Qfr ztA|SUshA)@2QG#jP+>c-)Ja~FExAeb5m6+Mc9U(=uPP^W`w4j=&geA(g>2}aY^^Na z4}2l+kZjBNH6aW;-H#l?4u4NP_QqJAIw`8tty4T>gDPuZ2q5 z=9Z3QJ8AtI^|<)j+F+J4Rg;gc9X>;mF3-TBO~64og>0$7#|2;GgJi zo{q!8bcA|z{B5xT{lS%3@JKZuh=IYjM5(+oPgTVvI?WA~I@6x{wutt4HG!^L(GBLa zvZq87iD%xMqx69`d|pXQx@(P3+M@A2k<`xM?+ctB*$5l&(Vi$Uz5zXp25#bq5? zjJM_w4t`#mPd`DVIVC!ObBew+}JPDQse%jKKw>nYh0Sl3D3%9_Wu z4S+R>^Gm_5JnH@E66%nO)}m);fFTDgIGiM5`=%l9XF_S2L%C8j8=bFj`2@r{DQzKkOwnW}Rz?+?wWjr=-$Sx8Z(Aj;>er3D0XCX4Wj7CC$SxY=UEE4yqa@b=9z` z_Q{yV_&t|Nesm4&c^{l_A)D7BUwM_hXdr5RI^G0l%7{mJTQzTDRvB`H2qEr>@!7is zfZq2=#PMIAYCPI_(V+hwciJ~0vAnC7Ls(d6Hs%P@JAqY{|HeW)vt?Vr-!L1w81?dp zB7mt-t>SO(Qng6sdk5Y$mzac<=6i*Ak(5zrN9bkTF4ou#&)Y!N>Z+ z(60t%uddXKLz}zNSleP40ZXLdQ)8i8CcAB zq9GYr5`9KT5KPz>2G<_fmIfym$^=ecOM|Pigv=6K#1^<`IT^UOz@4$QxI>7znJ7UH z;=$?vg|fsMJuVBHdhQT+GncqW*;`k;6dk~vlV#X02}_A{)$WX80VC^(pPgwX1Ol~f zT~kSF&Ioq4AjLps61Mrhh0;6-rMqwcclg>0WN8X#mYuE81d7Uu1z=rgOxwwtwJNr5 zG2ixaMI4Nx;Jg&0bXXEs@LPOCHaAd0c(rZNVN&pafX>&t@ zG+uznOB<)*)ij}Ll>*cbG`xVw`{SM97nPpF$mD7QDe5HJ*w@Mwb-9$`UM*#)zthJJCK z9vyVi6MFIRC$f2(X@yr-ofEXT^RYrDrnQo7ndzJVIF|xWX4g|AVg6GJATR2dHPkFM z^(C0Ez~MhJsnipf1D&MJ_;G{60UMp9K9q02P5(AF+&wa-bu~wsU&!Hy;y4X%HWN>w z*s&}oEU#fn-{)gEUc=cJmHaz%aClzBE~68`jR!PFVbi5x@DoDdM58|ie^{s zhysH34<+on1)YgadTi59Ti|1d1ld&9)}dm&)Mes9kgWl-Q_Z|SW=jPX>qWc8O%2T1 zEJnH7ep2)xTZZp!OF^{bZXvdiS^Z6e4%-G_(*$yIK-KK8qRuSrw5SLZ+C_RXob7p- z9+a!B_Wu%J+I3EYLn1<@2*Fjo|5&rpd-}&~9m$&it63cUKNdb?v0LUt*8i70%inPM z5A6$dLt4u*)vgULNxRis%HPn|_5TS>G}h|Oy_|^;q-c4cEOFNk%2leDpTZX6xJZP} zMFYjcX>Ooc$Y#P$Cc&lwd;DBb5o%Pvou{`6_RFnSfVzzteJc6Z_nVr~3!oWXFH{bLL(oCSnc`C<7vuYwv2p*p} zS{Q~xR}92`)^P)vHUgdclQxnHlW-aLJjm)CBekRYe`LL5aAo@!tsOg^j&0kvZL4E< zY&+?6Y}>Zgu{ySG+sV88oId}1Z@nL~s&=iFjjDw)$9R5ohAA{rC3uHm3C;l$EHVzL z&pgbc1M#Byg9Q&ptytMNPlJ4G#^DLk)g!%C)CivZ= z^soChtMHio%7abX6?>wljLx>8CyVWR3S(nU&Kmq!Bu6Bd4`#MYC)M(t>#?S)2x%Bn zE^M&yRWyA5>&o9;caGJ=2&(wh{yGYG80MnGDxz&#P&f?9i}*zBuY4xA^9J>wW&Qt2 z@OX|ULbXbuK#Z7~&hs})T=#!vIL6L?IithQsl__t+WLs-fM&ir$1>eo z1z6oyIT#qsqHcXa_dk4Ix8X?vG^e7PM2_d2-?|05mz>Pl8$Q-5zAz7vVjNBe&)Am? zWK$z3@P`aT1@ZNr9-hnf+2vdwjNm{;4vgSkEG<>y%;~b9AO3d2QXhAenE7@w6T-ao z!4t7Y|0a!qBNIN{6}WEx4cPz(Fu*|O+u^Z9^aAqhzptnwdI9;hBF8s|h~8`0tCM;R zo_Z_5A0TYwVeU#m3E9+t8b|xCa-~P%Rm@;1ULs0s;nE`47v2nN&12Rv?C7>LiHV!_ zRIhN=&ks(Q^>p1IP5uER|3<8T)qe@zhuqyA*sbO(gl9s}l?=;(#!@=JqWwvN3TnJg zf#;G1E+^5sNYkQc6dFw_&-7uf<`sk~hf40VOlNp7V%oF6Q=rVlAd-uQ?+`8Q#xjD5 zizizLSR>2?PKPIZWgZ|N4--uW_$2ck@V@*oK*`-wg&L@)dXU=Uo3*Au+o}^G2#YU| ziVK|3f zg~3~Dk_IiRUd?z-b#`kJiy|Pu%kLT`pjfKy#LoZ@E2J%dG;Zuv>Acb0*)S{`zT1@T zxn6!@Bm8!wAIff~o>18Wo3$&8DZqp5KePR~<^kI&6g}{$bn6fH+rA?J3Q&HXR?h#f znxCH?a4>`_&Yg$<_56pYpSidQ_s$Q*JX}U>)bk(a;=jd(xOWgCRuK8yaAOtK54%-t z`x9Wq;py{Uv5QoOhstB(`0=Hp?5&LFCAR!sKER?X+Wr;&wU(rZUelHYkGA9i6Te&o znb`sx)!;F!D#b1)p9QxTEU|DPDy=r?Usfr`qj|krrWv^B;m$jdI^BN@CK6%ludgi6 z)aqL*8ruTi_6#1}`R^^qj;>e14w@vvh~y^VkK@U*K=5n)#QLyXQ}ImYZ)%9OV7V*j zfETxowqO(tKcAMBR%l`C7*G(*uH32UT6+j*&cB^s!EHWF2v8xKhgeEXN)C)(HNsm% ztc_@QTA8EOXk^S{Vf-s_;J*a>tdYHlVSxV1E2D`T(nO>;RR!X5pc&BtbZ(=NgjUhZ zjiOZ(DGGTJ8=`=HwS>rR2vg}owSG?>A%EAY7+HXA2QvA;yVR8TFWzrij!JQM`GQx- zxi`dhwE0UM<}J&BFH^hy_sbOYMPLzB^4UMP|Evby00NxmfQ?WO_LlP24!m6la1{G5 z>J9+R31RE!*>eTa(hZu_SYA`1yE!OM#T-LcqhhYkW$uOr@CDDM{@#d^A@oPoe#Ea> zZ(+_mtz}!f?3klDim?PRpHStYEzS!vs{>TeZ-94VS_M!&zk^NbSR7T&4*=eYDIGmV zTI(BbFSv4(Sdp%1zasi3hI129ga6jZ^y0+l-Ve6_B2RzCv7~-WD%t!7QoFHV<7A3) z9GS!bSeWJJz z;+Vz{ya|(2(i(re!5*N6VWs;7Mux#-qOM6v(6X8>ug#i{2f^37Q7faKXN^8wdq9LC zPU4g|*kDDV1x=s=**mQex7D?5o7`cm5cj_vL^&>Hy94)|D+*L;5Za7~?oj-n>?eX1L5|bl zItT>3dD5qS?>rEM^#APaa0V(*FoBWZP<1Y2Qtkb9aZ4FA^E|NNF4syR6^BK`Fb0TK z>LW$TSG|nkyz=FPZM9B+x#r#?3&2{^O4JpY9vcC>WAqCMvo;1hlP8mpQIYd$xXc<_ zSnnp`8v{HVZ$BH>1QMl@RE6)VZ227z_#4DV+pwi9YlqO3=}2^`=2iqc3wF{icj(s< zKBInshURt1GP=!M?GK9`>C^r@E3A2bs)VfK70~d`I!G0m}_Wpo)XY`3_iqlydjt9(~r!qY}Pi%Z=~Q-&v*1 z&JkbG69}DPMDd4;mP-N%w&PG@OJQ0Ek;*jZSYz#W5v*h|eHWV&=f!Pn36}l7gF>E^ zOEoTlkAtP9K#JKHcz^O=B$Dbf3aE$k(19$TszsEUdH1%*I&RC8W_4ut7@>Q{*Xr0j5 z<7=j|%L=L4_vBi@iLbT2tG=wqZEt#puZQttSd_ST z$*VavC^biJ(Gccv8dC^a$-f*;t;7chEMx&52P=UKcnuFK=H~#xwtxpHf<51GJIEJ2 z#Ejdsf{6v+(=la9+C#|V-oN-DOIn=Z(7_+&DtWtK!2hQ3?;+3jxq=Q+RQ>yS-fzVv zYw9^^Z7*OsXi9+pCU)a3sUN`75_jH8CO#=VEf2M9hif{qG(PrSnv%NxccY%26}_rm zDTqSzyLHy~FU|`Z@Mg4z0wRq|zJaSefJ9f>bZgvGiD?i@Rcz3p8$b6G<_Fk1xxG8;}^T4$xrUnV z{W``|5?Fk@C1`^sW{m^kF!nK0i@oxk#aW+vwrcE+=)EzC&tMa$fts3jl=-4HhV4L# zo;Ix3bsL-6&AFzx1+eOJ9UfM;@O&InMzUNrwYfd?c!G@>pF9B^*<6g^Wi7c7L-UaW z6fuyvTG7FRCJ~o?s*HGxatBfev+D_bENJ?CY6l$+dD>0bTsOMu{%3F+{I|H1)o@v!}S$cz6~Y{g8R`0V$?)_C^QEKc-9)-)8mEQvdvc zNsWTr*)i$7f_x9G7`A7VmxXbOtZ_3&SIk@!)}ShDDht6RkOANczFz8On9I^E9gP$R zs;VP-2n?mUsFw&*xv(vfP0`!r0dJAR(AQl1hXSH5P>xXWv04X8ZLC9EXAA7@bAmJ! zuoXb=vmyZaM^-0Vl= zCWC#dVBU3z^`|EH!G-ygy8}1B;XBaiGn@+cef_nC&6qO(SFLe0Kb$TNR|3#VHvn2$ z`5%gDf7*pow7C?C^O{4L2`H;^R{tK)5*69(8Tks21mM)ww`X0=ih1 z!iv|i!5NNUyOtod7CZD-niWU!tcHiS?sWUM?%u!nx3NW_klUQ&(M|3y@>Jdo#F(2ZNmn!)Dl?WGZg zr}_)x{9#+q6ac%OZ&aw~b{b%}^Yz{$W<^u+;79uP+{U8AO^Sbf=@A0)k)>dJUoD1~ zC_0%TsO5X6E`7!-*p^Bn<-=7FLKkfT*DDO#OD)ivRgOxqU(~j=n^Q`0a4BY^YWz7j z{7`Me!OQy6hk)gQ2nd6HTmE-n5uzO|xW2^0$E>{|+4w;ygM`gjl+$;YyB?$n3tKn@ zW|6FihLc6#balb2hj*}Eg?fC;ouFyntTwfS2>1L7TOAYxGTw|e6PEo@baAv8)AKX| zn^=o1$XS14 zo?zWUR!t6M)nGVFaLoDoQr`U${5>=|&p606y0*=vXIT^fXp>0D+8zFg=DC)(iI&F?^}xQT{oEdk7k8Brxjx?IbO#?J4gtijJ(rTO~SZFVLLQezZMmcu>M}@ zrHT7u4DV9PV~kM;#ajGZ51QDOkD#T|pUNsaHK~vr;dhD2m|qrhBvJn08bxGdW{jif z>a(NGoZ!Epn#Nkn;xKyZ=M9IEY%6dc32u=~kIvq;^i2KWYR`Fa=Bx1p1 zc@m%1_#9AC~q15 zAujbv#DimTi1iub`d~w1aR4dS98nV0&4$8enwTv)#lzUQr1R|JfwnTv;#%Wh5 z(L}!_E)q^PE;o0SkxgX)A{5}6$e63XXpyAMP^}@h1KjYQ)kz~jy=Ew-730!j*d^V} z3PRd}rXDIZIcxSA$fN5bOdGfA-7=dpqY|wLkuz%~|HgBKL+;axKQkS@@q7gso6vY3 zn0kH7JW8{fk&`-w!=g3bKhi>D+oBe8_hz8A%HGLmU4MgrBR?^IJ*ul(Hm@rV2Y7h)hycl-n$zByk%tQdNP9aKfL8D$Z8>*EU9)t7m{V|mV^C((>wVeIuQ0i zzE3ZEvgA#G6S=H1+fnpDXg%f7)rmH4!`|E5`~5yyug$l*x*Fm0{dSSQ`t#-g=JhJd5{B-)#!sX%Vm_OVKC7~&qPz54J66yogxWRj?^Y*5K!NZF$vjViOjOb&*M!xE&tia9Wg#g5caesRmk=IS^StNb;-r&Lq!>! zGA{B*tEam#=G>%9^TYaB^_Kvn48`Y0q;su{5Vb&qH15Rv^pnWaGbEfRg3}K+hU3kM zINMTPA7$Rl=lk1_$L&KOBE9p^V+}Gsx8=!m4~RCU+sv(-u2X?J+>V>`?ffC8s>>#K zLDBjG`WH=OX^q*+Cq?*Sb=%Fimz(#VY;Vh7c<|XVx#CQ#l}=Ja-n~&02W2Zr``)*w zG>&v{mRQ{sM`@yT$*yI?1dsWpZ`(fONe{IdRKmF5N^eXm%=$`?~Z3V9xsoe87HRR*nvJ%i@fjwItq zG}k*LK-(R0ecp#v?q5Y7oQY&59oX{{2;QQ&tA?bt7pL$=sT=MeBO%@R3#2$Pq7wx4 z8d<;s7J*|`sGWg&n=McGu|RZ~m)2U?lE&PdvT&uCWG0qGB~7gU_oaX&Hb~Gl6_&KC zceIrEizGp#_kqsYBEu?{z;6!|EK{l=^pUcmIS$h^qA9eHwk7A?w3gQQ>4F!lE9>gg zL{BLM`OU^IX2mg_xN{;qR(3gZ6h)4#B~UdXjVl7pAO+J5ht0^>*FzhY1ph+3v?Cw| z^qv|kHrg{bN~e-RQGGD0-N^qildtpJaTqB9lotZeSVh9;`HGVdsqOK23JE#ap35}eQ zh;i)}GXPmIh!IDzZv0%NV$)dyU`aDFRI5C%G|sPB_<0_rieyNfFQwysbuJ*Q7vHv3 zQ4X&|A}H7R5C(&Ga9m~Iz+0?f+Td>}gmfsPUP7>`*Z86%gLk0*+rtQ09~Tn;ZqB6V zsa0PEtDZ6px&u?(KHMr`P@j7n4;au^pm?7Of0D)Wt}f5~ZN*Vc=@z<|JEA4pQ9~(9 zzxhXP@KR@}Q^iq;w%^W%el&_ZL;iR0I|ch1BYH zr#Q>@ss!7Wt(O<8MpQ2Fjc86W(ow&B;doChxerR6s}kU6o89g#H)DvRRz#KhgK#+g zS7k=mizaMSWOzqXlu0pU^5dYBnQ%0Mm|!?(HtZC|2t{$eJbIa^WHcy1U1z4V?UE5T z$hJ}Y{R0qn@QKnmR~2_HLDf7=mDGu{vS9;No(7#ERn@HyfbW$;B^+6LgR40&VW### z9k0`)dI3Y$9z?z7-SWz}xSqV$2$5{So6gpHUPR}Gt3glH;$rUxKA>Rs?5?nV9Ob8s>uCXr+i1DL2t#hQ4lgtUMU(tzyPX{+*yB?k5 zPjfuO$L2r|@<)?E@+C&B%5*>_kEP3ccUMv#RiDM1yHGRZUXnZ`PPLZ}s%(iNv>j?z zeJwVwt880CJG%C9?jOWcoVNfg{fzXkPvhYkUTuf99%+nZHtmXEu`cc;`ZD=OKut^& z-qj1~zqLu4;Pa;v;a|(7SRqpSN|renyunAivrX2e-O+N+6;iQPo2@Fos12q&V7el% zj>9!l`p7p`nw+&~^`gg_1ycGbc6Af-Vw6%P!%-So&d;Xc&yEwCJejzJzPlDCvyV0k zCU`4$@#Y;_@wp>g-GS^-+=(-7eObqcC4#hy#M>>3dm+ZiV*bJPzM!6j%Z7aG((=&TQfeQN4B6M21Dxxccx|1iMK{`uBX< zTzHaA4Q0y`N*w)E|DrRX1lyDm=1~+(<3VgACcSh6n{l048D{E>DHzuygFf7}c}~tR z1XkDe#JhD<(&KHMV2MT(;}{+%t<2a=yh?PiJezD8uvs?96QVwFGh+?F{5z^cAnmq% z!sD?t%XQoK)hcMqu45b5DUWiF=dG#q!|iS_)72Peao%5giu;#LWNl3|bn#q5>^aSl z;&#uOG_=t~i{gb!-?@!Az+EmgM(Ao?%_2VF3RLV#zAcB96JS;ISQhcfnaLE$8B|P^ z0Hw^hW!fWh6Nl>HH8PXg6tg%Klen0P+)b!la|a6#M=Vem1O@!y-GiWnZxwV9wqbZC zHZpvXLStsHs`gXDe~!>X+f;wa>%i!qY36A<&mmZ#XI`mZD^p>?OY{!&HrA?E9#WP#T4s(VU;;SPE5RZt z-9JU7;@CJ%B8Reo6Xj-;Y*Tvt3r6+n2Gpp2^3)W7HS)-5oE@1mh^+(DV+9KwH z98vYcX-jw)=<||*vx!?&lF2tq)UbjK5m&34wEE)k@0pF?!Ch9~wE`tKY9#%X>kl~) z?WZfzlR3K?gBF-V`2>;+uC*)IWEMs%h!5-C?fL=6;kF99LoSrpVJU}dcgfoT_#&O zA0{6QDH7}*7ja%}GDqHDjGjoTZM5tZ$*kj6u}WW%Q|xEvTXLxk(z2`0%PJ_RAoSBL z95FsVyd*4E)C|!Q2%@y0OwRF)zcp@g90d%zurXXJWO6YaevPY|-C--s!U7DsD9)Z$ zyD2?TDG?2zWr7|pQDY_mfU_mgmI-{n(rwK6z_Lp0285R?&&!L*Fk(_cH_nSFYss?b zN^6U-bLJE5TM1O8Oe~6{WE!uu_dF+fj{TII&16ECc4UQwwQKl3e0IXt7@tPIVzYM5 zjpLzh8*S%VMi#;nwI`x`8H?tIExo2`HIXDmvF`VGdzB_O_O?G!ZgxzSSxeX=kzLqG zc+Kf(Y52+YeHU<4jSq4Z!unpRDjcLm0yOyGj=&*`S%oJV2rSLg$gqC6p+MW-sS=pu zx@(ZnGLJRoga~CILs*6dw$u-*@jdnc0GT4I)Mm@@l`ZgnEHgdTfBxV@KRFnZo56{u zV<-w%V5a)mj>@~#_c1i-Yzp5>ei<4mRGwN_*c^5(<8>#cZK$DhI;ISznKK0B$;Kky zKr@no5z_+?>IE~_E0{id?_B9={;KnoI%X^xj;=VH_?MHO1@>boQ2O~KU*hgaWA1i` zMec)y@`{a}spK_J`pG(;`%&C{7?M^(rj z{ap1&Rz`Y0e_Kn@pX#IO@f5S1OZ6$;QZ5+tV{U@wPAqvn8D3fE&yuh5vhqNa3y6F@ z>glTHX0{grsPT318rw4VqB?0ityP9+Uu*?mW~r`1muK1g4&RjV}KA=-E8&H+iU3jO z#B#lS;UP40+Pu=APg1j^#y9b|E!UJ0Q8t<@5gVH7kX6aN1T{fvA}LY?{E%tkcn=Ue zY9yIgH@IncyUEsFu_ED-V`00p$55&n){!O1ZsQ*XJ;0giT6+(fgBzs?rW+}M{mr9Yb$imUi2tuO^P2)zwXWMj?Mga4TqmU>X zw);7uCV#w7`dLo(m!V299r|=T-FRX~?i7R!k&_#ZNe0HIn^N*up6u!lPK@NmW>Zmm zaLHvY8nFs!qA|&EFQTWm)J~2cCS1b$Uhy| zSg+Byswj`xhV!B9b2(11Z7KkJ*wB6w|F>r_0YV5{j?KGf7BkgZx20DEFQyZFr1k_` z{xw#oZPoISwZwCVq!RBE_l%Rc^cid$S8Q#FlZG;T%VPq9C3_Rm2#L?B0M-=vr)MCq z(q2zQ9qMoz6yK~<0PLEL0G;4wMB4RhMJTdVfqI^CY`L0>)xI+0~hwQ!u%n4&JgqWO31;)qBxD=1wWuU9YC5> z6WV1ng>7Fzyh}Onc1o|6N{o98;ULj+BZQ(F*mV*VB@jdVc=a$*$!Sp($yb)cN+_nv zG6nV6Ci*Y)3TTef*x=`sNqf%d0kZgh;LF4y8)oKm$Vn_^uKxIW6VgK+|seCfv$8+ODUPYzDB#r zQ~@<%o4UMNi=`p%rL1+Lyewx)M-F#hMqG_maDA>qolISJLaV2OYNX|LMlYcKGQdU8 zVG&D5O1%4v(~p#zDenHcxX_Vdlf;BsX!1@Tv74A#%UP4&`{o(&XmoFBhzZ)(49E;E zLrw%|%bQWE+zj)o5#%r6th$cNhHWatduN(OIWlKwD@CMFsmfX4S>NngV0m=A3e!95 zf@RkB5=9q|!EnR1OB{;o<-2%0S!AlSGd{N4&w!8_OX1R_b#Ln8CPqL<=(aQXTAxYN zgp}awy3~EJ-y-6<65+B{XN-?{oDVN-g6V@#=y!B|pEwnZ1piH($uU-GGkiPS1 zys9bm5seiLA}N-)|*v5BX6 z$C}*{#_9oVkMcr{X%~7C4W2>4#2l-AT8ZEV!=5;Y-*c5AICtFKRd=>j>PKlD6PzMc zbLk|X97mhMDdow`<(T7G2hFYLrfr=or*u2I8>5(SMet9E)m&&x-nW>RQN|-l z@X?-yb;DiS4^cxMu4z`2<6eRr1GCXS103;Zj!=B0gLGCN%)4hdHOvuXobfR@w#hv! zbrejQC{pK}n0=Hro`*6nVV&GYcV+~IKZ&nL<#fQO-S!ok_wBiih@4@JoW;O@B@CWM zgagJC@73K`v?X3O3bQa8K{L9VY*C zu=BE%M~GC>FPNVCx_NH^Br1lw^x99}RAD?4hQ@s;W%w5e5k3W4yk6|4)|^RYJu^#H9B>UT0{%p?S-ZKH}* zv#OCc7e_UA&8jAFKTt21AlWBr(+-@^;MFxgtuUdknY~!VlDP?y1<6R`9SX~_B@LEl z3XLPBpZ(H~AN^cx`f(0LGE?|ECy{KUA>VhDFCaX)+*tOU5j|!+mZX9lBQqbnD>YNb zf`wJ)FS3p$X`Zdlnd7<5*TpbPFr6J$&94-(HJ9O*k{taj@PeFThB2$i>*+^v-dH+m zGeJwOa*QdJRXt$UByRVhJ^PB*{ui-`v@K3LSzvHUGmsxPX3)#CM*0)K*`{s8`83&B z;E`;|%G#Sm(@4sIzXZ!ypimbiPo%mKldYzn)f}UDNdfXScn$s zd#t2_SYsNn!CbG9tU~@B@P-rjE#Z|&Fg0GUjkO*tYmeV=oGTl4*)Z1U0=Y=mb)d?k zh%xY7B`RrdXXCQqO`=W514+n?HeMsHel%Motd)TK>qlhyy{I1fkfoCALYs#gD12CS zRVy0ZugjHyF>cf_vi*QXZgr%!P8F2oAN$gk6{s5n^3MmsYSzLY@eKB^!aMnZ`NSYj zSf-$9o%}$uTUzO>lq{t?jG z;C*_4b$(Mtt+`WgQe(cf_tn?J49g>Z?4mCOv+CovJKNj;9N9KastL@S^~ghAX3@8< z#s;iW|AXc2{yAb4A%Qt}<%Lt-m<|vL3|4Zpr~dKRK`vXmy1G7I#^~3(7Z(>pKf5|U zPA2K~ysme~y1nl2!_gP%yM3+)?z6jJPRgshORBOBL9~oYKXyFs+VOnW$Y-jms4ZOazvvVXPv6MMlr(&2{sHbF=?Zznyf z&H0-+mHku`^6S#_^XjrRW3Xy15#+u)FK1tz)bqN3ds+SpqQ+ZLH{1X8n`P_GbHd{a z;HZssCZ7VVhd@v`qb zOv9Anf4uzTuDklsY8l}&;};D?URkEZ{wvvs45c+Ld$m>nuNDp2N2}V`$PT*_a;?h} z$1U;@h!!fUtHIncu6Ug9t|n)b`nCz*_Mxq~Tv!aYvX?*rDGx3HMKn0U(91H#BIfE3S2$M1s(6+$Z{QFe1G_ux1o)1 zGYeu=Uyf&WNL!afNfX34FQOmY41H@x=Ff*)rUsKMB0D;miW*S(Z^QR%b7p_$ZzA!R&saT zI6n_4I=j%ZP{_kolTZG6Zc*pH4_kpqKZdHK8g_-n*u_MsCQCeAMoz&HPdQiS)uVNMcrDnV=Bfjz804Jy=|Du~+`iC}YCN>6tyheay4; z$ZW48Yx+FQB%d=&H8o`##dgbFpK4alxlbh*yCj!ZPtN#uh?|M*sydiafII*dFwBPN zmA9E{{Y9V^&75LUp(4S*0RIC=oQ6uP6&h$!B0X^RRjf;cb+Y_$dK=s+;fpA6SjjzH;w-qq! zkN|_p){_uY=(Fer|C5-hVN0j12p#SN{hlEAMQz$*bUsLbeW~5H_3#R5li8}IHTquN z;@0p9@`+I=CTg_ZH=|^*!CE~GdzUJ558IN~98i~ker|uhjrr=nU)%aV^>yp%y;pxe zCG)>!A71c(9O8dI41KPAOm@FrNB`OVc{SVpe)G`&x#jy-?VEh=G5@*g`@X%o-To;7 z_Ph%?{IfujHTgm6xcr0qp-heEJEEceyxhAN5ups@un|(xiZdl76mt>Km}E*yGY9?9 zX5+Ov<2B_mHThFFr7Q=t;PcC&ghe-8tJtZp1Y+!ieU=U>8R2RqUtOIbp_>k`es`fP zu?bwn@m z!yB|h*HbDBT|1p(tmIZ-%@TtyFDs0b8iGhBGI;q~=(=TtW$*rT%y!lI<*Ui|sj~Mf zqau7{PR~h*E-2coG_Oc!b0Ur)sER_%1i+1xJdQA=Hlf&Gi_fU z`W?rj!s2Zp3eQ*S=;&)a-;yAicVnN7R@8QYsE~ZFjR3Rdj?{+ zEF&nXQAav*7HCBxeY>5qADhSXq3Y_N_|MVX{8iN-=+DugnCzMSLNx7#9>5FUha!&b z(}c?TZ@g!rH;-NTqk0M5xmgsx!sZmdiXUR3W=lw7%NpgQFvv_?D#!LkPE?a#dvUUu z=83yzlTO^l;E}2$XbU_3{gsbHTeTGH*3zyH#jP#%UJFJuQo&xHukaSfIfS%{T_Mw# ztto?&6Ti%4H`$EKTTr7mqrTq@3Ttthn~6%7en)M{jHq?6@N2}{sCK)hHs{OU6N}H^ z92sm0Xn%xGr|}LXwXPbWm)`qc5|LbNTW}|v;8jah=Z?qk_4N+oE){H2vkPKAX|GQL z&d=?zTL(#Uk$~b+6Bmy+jm+S}=nkzTJHF++&ufpppN6TH91C=gI6CW$b}3Xgzbk|3NU2&}>ZYBgo9 zPbH^FCs`vYX6sUv=yX>j#>g-|q;PAQpzA>nGG6?BzMj{7dn)L#my=Rx`x&>JVbpc; z2y}#UQ0V+IW!SPHtL|3+@afmA<42zOOV_MSowC8TLz$fFr$$|!rh4rrhfd-0$E3P> z&yQ^|VmNX4P_)R#!7}H|h`VK85b0(9k0eXb7cgt^tP0ACL>B!(#r^2ag1Uo{39z>s z?TgFS8oORid?vHGnbUFjG4|x5UQAS}sIjma{zD2Z#axfmNlg6n(C}$uMZE0@{8SB4 zmHI+khl!ssLg|uN**iEGq{B9lAN7oCpuC8v7%T1s?AHcfJ@TJrYf_lnW2j$iV!%-=2ge8H`e^8Df# z``{8`)omAriDa&za-3psSEJk2=w6gKi%v$1!Xwi~4x&=0}yqk__SN zA&goJazn{xbehv&-<{6`mVPTV}Ghn9~CTxqYr%5(CHvC~C2%Jh->+ptltJ30BAa(8>)>=HMpmLV zl14B8IYf(K4S5q2`&W@cFf#_hbymcswN~p~M#^CiH|6I^p2V<)vPthPphYvtb20Eu zr6Kjyo~&I+2gl~eTUG~Xd1nR3_VcG~D{1!?ga0$)%Kb~!8`y%QEb+|~g0HvleZCe= ziw@7{MN-&8@&qMlar1mq4-m_qxN4kJVHuYLNwprLZ_YO&zt5!FZFt9vR-dGxTr~o5If-b-6XlO*oBqC28$RswJVGePTdh}1Cz37bn!QT)R9P?r zCvv(Q^H!iw=sS>~t^*eqJCzG`&f|B%{ZxFt;5cn%ZK8>$B+s9WJGhmFg-!O81g`JIH9t zb&pst!lk|Tloo6u>c-%%{rlRPbfB*d(>r0s_xM{;b1-nRqew0L2j2rX^DjDO`Xitn zZf}mO>(hHW=RZ0lF;n=)=6x7st-5_`th29KIElDsEA1g>9T$jp^2eid=G9zkkK-0vgd;-ETI7n5cpQIU7e{aTcULKUeAs$8!d@8ua{*< zHcP9Z``q1^);i38yr%QFr@BL0HFC~Mj=X@y#nhk8KE+_uS&H4K*1hi8_KdsCZDiX% z1V%Fv=c887xPeWlA*+Sntl~Kq+Lly(Zinw8RG+rR!Cm434I6aMwyQ)Dbstxf@0m1XbEI(A!r25g`CN7g& z&MS}v>s{wz$$3oc57X||3Dz`P*iQN?XH__EHUGloI7ntavJr4Vg(3dcV+HZ-F(F({ z_?iJ^_PiA%3j8}B3ynT98S!$%1Nx}E#$6vZZ_ruCwYsC_nUW%MnYthAa36us#FN` z?)5QJpwFQt;lhbh`IHNLdaKV+^Lxc=8gwl*+Joz}#Ax>Yz4y^^m#u6~PuLJK*|7bJ zDiT4h?u?zz2x#J49iUqAmyKHS-F>AA461QLJd65Ca+Nf6_z!ZLmOa5VNfz_e8!`Ba zIx+aIG!dyZ3+2ZF9k<$IMDo?ms|W7L1Rp~rotbJr=uyTK$;VwK*+X4e^iL2Z7`N8d z8}2NRUkz*#H!NAl)NHoVc;Z+HX(P3&OW`nz^4u-r&LA1hq*c`rSn?nyUzLaH$D#}< zwx36@oj=jqD8B*u^7b~=7DiOJRqr19sygBJx#m@^B_~wO%&Z)2j)r5?t5QHv`*XEF zy6y~HnU+kj@G9qp(IZ;N8Oy7`$K;*u7$qWnLD!Z`DZO%u)VY!E+(>t;ZN54k)6{yh6bJo4V)b93KB zpRsZA0Idkxl3w9YwX)1Lh#SobB%}+gduC>{LbSJAyz{hymo2viVv?lv0t<`f*=cw) z!G0DNhl^u}f4W7?%v#M$d2;#j15x&DP-J@N$NXu)rYD5Y#OJkrMay3n+&-v*#-<>x z?8RP^bWj<>dBi?`Ft)SU4VvFybY(aKlam|p%EZzCt=scWmUb&v-TJXA`{Y})!?iLEQ@d;7>}|AS*Q$2Q>oN+ZIIi?o2T;yz;}q$=f+hXl;yQgm#!Pl4VWvw z_CLq@am+iT*%b9XG%i~lKw+GoS`3a2tQke6Wm3jk)G!ty@@Esu8++PnT{mB5a)us% zV_D+eD3SjT-^M4f3v2YafjVHOnO8+o!EJxck0ABoi?wsMJTv*SScpS_#d2Z935~ql zo&EYeAze?|WrALglzcRVoKQ{9r8yWbcOEz(cVv(0?#%ykGi#%oI_SFCAeAL{QZ?Y@ zp)GP6Yj}m1{kzOP)eJ#U?Qi{*%3O1*};l#Myj6_|Y2{X!g4iQVsS_$T5kafX<(1E?q6WVvwhyA^?G7zJ7i-WMBwfs3kL;o#CYh&dIP!LvaawYorh~Q_Md<9tjl$EeG|`f z1-vPsyqTIRy-js@Wt~bhnrE!wO5$%yxK`D0hAWRU`1ASzOAWWL%qSH8&+FRX&0vX| z)q12zr4$s8(91b*vay|vt?2^VjL?_~QDH;g-M_h$T&-4gDz_$Xr~Rj&L=KO+pa`4Q zqM&JpA!bqpkYG8u@du2+oP*cpT4?Z1G(q^=M6%YhYUF}b)prYmJ?|Rh&}_nkd92ga zJu2_CD-U}1vMtClb1_iq0_Z~23$q1#f$c{GC>6DE6-4%-@stk8kg(#^dWcOt;j2A5 z3mkxL6Srve_he#?D!uNV@?C9THAh^fL=%74giR2>-x`~C9*0_QiPNpF`m*?gg{r>T zaO#HKyD3~7wQ*HF-3h64tXS5!8|!c-1#|N$f%zs)^_9f@~V>2y;fb z-mw^Rt$UQ|*!1+UuSV=#5U@5VI*=>+pk`I*c7SM?n_iHL5M)6?Jt~G9HvBSIWoeEYmi1Y{N8yi^c5VYtV?0V` z-C)RH%BG^lIAXlK>3#A=Z_^R@U_#@^w_O>IknL;4=x8^uw~PV2vL*B?Lj*+OYK+_P z+%18CgDVD?6mxT{MC#7l51<3|Mv!Pr9Bt69B3QRonCZp37Y(`L1}YWk^6VPl+rD3Y zqw`oam8JJ6OR^lN5~Vbe4Do9*ucacPKXjm!gD*36==#kgkXrYz!&Ou&&bLbBoGP=nBQ{O+Yoqz~*VKW>D3j7VW$0-{V3T z6o$!&>%olG%Njxk1EAr8BD7(=h*2Q(eq~``=o4L9VJiDiLxdhvmjRFo_%b!x zfU)j>w|YLi!$~8gvJh#0Z~=_!HWArv9MQ5*Uma_Q{y&WQ?d$Y^c=`X!m{(;4yu5xh z=F}9w+0#!M0g}CEJ0pH17pJGEE1NHMytIG6UG4^TK7)@1zN1sc9IWYeryB2#?=Esz z;dXEIlN;j58mtk=~C{`RPa8Z>pRCvD?B!mqAu!tWe9cxMn0y}Pqg-~ zm+ty4EmUdWpsb}#ji=+2HyB| z2SA>k>M!SfR|MxwJh^ZE>1xP(wxLy$j<2;Hg_B#ZX@DqdEbB8Dj?zRB|`{O z1mCtKE8HS$VMahcHaNC-wEaxk_<6 zNa0kA=);flkn-^;zvwdGNF~O9*4tBVaV7FShx2;${#;^TfSkYBj2xc3n_#lm;}bnK zMbUi;dECVLp1CTg)n#v)615HP@P(+I()iyv$czPn-=X9oD`gJ%B}IJ7YWU+hQGCGa z0?t0o-~4T{9bsfstbXthg$7=tQ+9B%Zp;Vmc^f3v+MT6)Xqq!v4QI!b?M`7g*(=;}QVH3*Istt`#;Rg=C2*IL;AJ}S`b2WaEvjjjJu zM7s+o@<)c>gX}y3cL$qXhu_=0XA1A7lBSWp$|GYd&Ph9yZv9P~2Ga5=B!$Fi$Y{18 z8@dzaO;+0Dyl7WcDb!Pb|6s#x*bGKb?8$&@^QN@dGrvpA*>on2xEWPwtnH%qN&m8C z@%4WAlybBs=<|3sk<$D52y6`KIsQCKnC=bue0xRO;tTjZU#Y#C_J8*LJmkE2e=?Xb zXTJKne`_>&$=MPjn=T0WBE(jMwnk)5X;z3hfJiJbPi5{?_Yhr2$%IT1-Uq>BUGF?U zxs+}GX}M#ymE42Ucg<-u&P|)C31Rcpe)%VssPW-dtWwcV-GN)nWkkn>P_2&KYYDBl zc}+xhEi{h!K7VgBJJ-tHinu9spVv68u4zage5c3a%u-6i^2}aJEV5lqq2sFV2etfs z{x60nfW{lWysl#iWRh4zAF?){A!#M>COxOE2E*fe4lV|kB_TP(AZu;59(A(z=+yaj_W=1QR|)et#hq5{)n^9NtU7C^AD%!OX>l+S2!b?go zB9sx1l{!6JVIO41AeUG-=WFG>$Y6;|TFM!+jUz2=rK|zzBzqPO=pmeRxbfetM-_fU zbD^C&=fWzYNhtZ4T)OfKW`t#?_y`(Smya>YxSz5=LjO5&de#>3z7C}*S_6``rP(GZ zqf{=CKtV{CatxBoe#*-X|DFIRmDY;z9v6qrh|ix7 z3eFq*588vOu=dhXWTbw)8d?JLmQ@3%>caIJlwd^_gXNZA)W&Ncyh zV1)oFIK_FyDth>|d3h#e%Mb57yZAXZ0M=bkXDkPju||>8D3`q!dp~-R*`1hjasYyd zpjF_-U}bDcgYru%K<8un`+X?w{g}DrAta8{Vr4(*c$*UP!8%T^DEjQ?W;ortXe$pW z{$!7feO(HrKiw)1sF|p0#Cf`{r-)*(PMIU_zmwA{AMH9wio=@?MYW&|&#V7a5Aj&X z!#H@^-5ax>UeXE0HyRh#Ce3V9;4M3cD}U>L9tcLqLo2FN@3P-VdmkNjK)M?9Vxp5dRz-ZSfb zxsc~e_uDD{x_IQg!OgTKO4Jl*I2UZw2GkN+_hwf&8WjFPjW-H>WJbKr- zjj?JS4nx!tE3*?!nY+YQSes< zXIY+!854@Ogm?({`Eg|O`<5H7q&UXC{#E;ih+Kaaawemn-dB~GF9bF>NsV~;q@WdJ zppO4t>qag;85&N}({ZA7je_S1Rt-`Dthc+teF-*h!)2S2nT?O?vmSaB%=EG@?fT}% zOe?ael3?g|0-fv8oB8?iXpAIi4J2sv++EYpaJyvEYsU6}PuAz(_jndh??h|cK<8&J zf?Y0%RuSHpgJk^m<6!wJ!r+zbPq(iWXeIlD4_^UEOP&pfxpJ`*)IzZgO=g+RM1%Z$ zT$4Xm1IzZiAzb3nDqQ&A^sqIKfEK4>43WGo#4ufeLNvmK@nY1p;R)K z_Mmg3>+-Ig(0C)&pE&DM=jyV|c75S2{KyXiv%2P3%#SQk&6J7mteuVErNQkFa!9Gb zh_!$BLR78tY@$x^R5y(EaU2yVLE8TKS8}u9PG$cX$m2(EGVDzE2Rja1>`Bjp7V}9v zW5RSG-nz`ln!8SjQMG>U8^-)B!y!P%39)na{+&#;s$2&B)UpU$GA)C2Z%@k^AbE=H zG{~7x@A+hK5=13NP4DR)X?hDW7RvV1W_=D^?axfk^SzL*sHmPgQfP|+KNkvh_fW!; zX_LDkKeES~jo*CE7-rWR$b(Qlhquu8z!OO{T<`}Wbufk;Y{NGxgb?p*6+uKcj?zE( z-Gl$K;Vu}E*TF`hM5kDZokS()Nvm=BY%DXoI~0EOYZHeXUKz&|Yet*H;lmGUlV0rp zP?DMF8Dv1Ek8zTF$wEh9e1PWm1KQBbwDk708eJ0fPs_)MN(0$mUJf_03i_~~+V8>N zjYy%7E_s@tPaD>a>;<#}M*9F){ImW`cXwv)?VMdkUQ3n=OwkvQ;h>m1^&6XqTL~Q- z`75>YWKSSII7MXzZaL+gST4Kqc^0-~JbgVNusoPwWTy!b9|@Aj4nOzm0pu6M{*UH3 z;I_@&#Dmd&beTU$pqDduAhOFk8AB?V9_i~#dIOs;>Qk{aYl*EGmec+MNR$3x3c+Th zY1P?xhHz`nN))Cww}U z;f4zlL$Bap+69Z!TMx4Z%huo+x@i?)sa46d*blbhX1Xuyf6PWNMjSTdUfMCWeDvXa zyj;oCa%lva&ieXgE>pDXktNK&f^sNfRAss)LREQ2{+kA;NY5ZcHd=IEJSoit?w88~ zg*I*E_kny(@T%(Qw=Ux{TKbHpDrFo?0>h?5gST3+*mAkdLwEC3SDJ3>jG#CfT7$k}>&UK_M7>be%)Dw4)Ej+%z zT7Kjr0u!o$K39=LHHV(bmP6y-nF$wisr77o{e^?cR+|Kp+Vvd(^XLWt zTuY;}lUjc|5ALh|)X{-kF$;hIR0$?8x~&HP|y zrZ3sW>e`jNdk@L&p=H;Y@Ea~MLf}uAZP}i}Hn#b3tc{{$g57FW?y|h>SEv2mb;#|b zgEA}#U_s&4I_wD8$H+o?&M4f66*8F^3eF;FQa6C?e?xocur~o| zmz@q9$DqgjXR>Y-Hkt}skIU2=eIg2i*W_gzcv!l$Nl89iQT@D?W<7&PZW~M~5Q!Et z3~l*0r5qd(3Lvc$miqoJS_u;O!0-dnRQP@HLhL&g;zw4&{zrh;;c& z#9(75G`@HMHcy&Z&?T*YRRNlCJ4s$Gh%_KbvZVm{wNM8}GQ|0B^wENV)`mJ?^64CZTSf%{Nq7JGJO`+@@IGi?EP=qNx5qkrZVXZ=o zr6ZPrMsz*fKJJMvF*p`M4H25cS_HQrKbD^?1~VIt#xCc^kpw&ZFwfTn-dsw(}h zF*Vkiy$(3iSvj`B9;QFE z@;p29raN;br7+9pFTcl*D7K*4PNzNibSgHeh9W7B>#9{5d*!kNUPJ` zId#Bib4N=GLFe>7TBibhhv zx4?*whQ+F#%b@@5NeDHMI1qz4>$1o7@cTy4~4j)%U~TLjK;E&CjVouAyjM7JYCUthAUhfbEa2eI||`Ga>Rj zR`muejmQ34LqB2`b2{r|3$>Ss? zAklftbR?6ZLX!~-s%5O|3hls~n0-@_N8Pps6ONxGWgu(x>kUdi2>Y0wdEvOP>TlO7kqQ_Vt2bsQRVrUo*7?1s$ z(00X?c2*A1FDU06OKAWnO7^0e&t@-Uq2mPlP!#*`)umpWa`Cbyycn>~?gJ zZQ!qh6j&34HJ8`MsVLmkFFPtV%kMDdUFro{4sjZ$IX{$X1rL(B}> zpo@oEC$)L3-Y(s2+Y6$M*f>!81N+=YLcJ7_Ojh0YB4u9^x^9Sm{^KDxKKZrL83h-z zF_XMC%|D|Z2RWL-+-hslWyomxdKLpo;-S|xT4%Dr$+)^-a0Cq5@*`7%#Z9kO)k1meTGa)PZyZTm^KrZsMY8JT&$6!wqzwH`qp z*D-Lls#E=zN?BC}d#3o6ZA5v!7eR2np-ay-AFfSf;;^&XJdI|$@wUxusaf(|!g2H} zLCdP0#kquCGueq;Gg-JZGrk_Qr=d18&PEceZr;R_kmi;!Ki-5$J(!Hg1NIn*oC<^x zk3wbR7mf)Nf@m9dT}=PiZoC1ZVN%%RSMG~<$+!&e9SZYSOgo}cK(qnvfdb|I=&)mP zJQ-vI+3^x76ww~+7zcur?C#fBizsHL*v1l@ZVRr5_Ky~9$J-G^I6@D0J5L;9+;1uS zv4N1O@#}aHb6vVVqHY!LxQ1)ENwal`**yIG%SQ<Sn6ho!A`thsK91P-5OXU!F?}@^vhV0t+jO;q7TnEMnL5vslHOD! zKeaqE=kleMlsbb4!7}Oll*SuPcV*hxem!_dr!u6)Py|jAg&27p*akiWih_^)e>Th@ z|Fbi;cQkz*D0GOw1zPAy-=n<>3*kgwy|n{X)m7?Qlf}aELS!Tc=1D_}+jSn&EF_kRbvZbLWxq=2mND%h&sKOhLlM6N ztbyT~2i_z2wL)uMxkShMt_H~OfeSwvpXx^X$REb$U1fHvl|Dxe-?fkZm>UM(PuiXl z8$3(9{fu2U^b{+o)+4KqxhD9tD+lrzb0G0Uwoz6f_En0;xI(CNAm3X#aP08CAFRV{ zfqpI{Y=LXV#<+c-Gt3CvR)0X}d9Xqdz2D@k-(IE<>bxu^>s08+X_S7-v{RVf7!30l35}QCH2XBxDkKfQgr0j>>55k2#oDI34TJasVkUmPb&k(x&7#&S~nX`N-qf3#r7T%9CT~44pU0Y+x^X~dm|5{u?2y3 z65X|}MOyJU$$A)~Vpb*xJq>SlbIZTH~H1QQB5ul+t$a+xp-SQ zW1$uytw>e#O!p-9pDlXWbKg(;{BYG=lCG4)cymqOws>x$K~oa2(2yLKt8buQW6Yao zR)c}&g8*d#cZ{2_|6P_C-Y7&D2RWxz7Uik8R%9rQCRrHH71{}iW6Dc(p9lA}+sRHR z8=<*ux_8zo;%;r3;-So81&Pg`VR0#(zAXO?DY2@PMLYxz$z;&uZQ1(RG<3ta!kUGA zbLKuO0QF-&POm?U38ONYfJ7TeW}4(HnS7}l6H65_@Zk~CRwd+x+&h!jq6P?0!1mpE z4Nn-1XIm*LirFy48JpKi8C~^W_zrCb^fvx=%nrD~AKkjR`em{hUq9CEk@7Y^yEr1_ znpPc_K7MhOEsyVSv0RGpFn3~0{KOP;Lb0Il_5@WAM77m)N2qS7(-gl)L=g+ZOUX~fFGG?s$;f(Ak_Xgv@KX>w}tF`OD`7)7RMLz8T#?hQ!iNVMl zFsHlW#oNM@w*U1K@OABg5|4&X+lZqR_N-e@ly|MV#{< zmUwbnn`X;aYoOaGZj`%sFpX_g$9s zR2Ja#a`g3j7w~bgm7Ec=99X(2_pv5>)ovfK4jz<}k|KECw5p&<{)`2jxTN?r{@i+i zC;oDIDHEI&&-$Y0wzHe$}Gtmg&LyvoKXkw(BF#`O+*%Fy~eW zxc!apnNsb+AW6x)y;tF%O?wXf;Kif66hG1zq<*8j0ol*<9B z!WnQMK~}-KGQK@mRN+S1pwT%`%+>BhYdh4zW4GYiE?jWOqf#VG^Qm*I(%803lavyd ztvPn|Rb{p!j-HIwUGDEBnouqA5aHy1SLVS!zWt|Uo@8&p1@QTCnJT#Pyt9GxrZgd_ z=LGzmcd+2tgKtq6Y}h@hy7MUbboSb-ji3uD48H@XX)i`G-Da@SM0ZG@_qc720A?0q zu0r97u(EI1Eaud>Pl_J-(bqli5sg6YhJ+xmeAow$#O=r649%F{RcHBHt2<;rc$>X3 z-BN0Go1+J0ZQvR8h9JF@`d6(72vLsZPu)}6d?EypYs2o`IM8&?<~u8SA-7hL&2*U> z4bbVU8S++@8DB<;Z^kbU3{~!_EkgQU2J|h|g3o z+VHU(YGeoMUysG{UPP<~_*qmr06kXkrqolACR=yw3+xfjEciFR5?b27%Gg`HdQ8NB z=0}4I@9zS#JZU;Q+^E;3Vt411=Wo_#9d)1t8DwmTd2;Tmo(~Nn$`u)h+ z&mT4AMwy$#nLbp-5_dxP_^aMhu0FygaTN$nblBEoOX8g|{+qMJ$s~BXD3)?dH2}TPZy=-`F(AC<~QNBfMz>iPhMEu4J-pKzUdm9RFmw>fl$`^`zSawZ{60T!y7S`g;M+WS`IFELN&X!iQ2MPvV!n1Q^s9idy8e8}qJc>jkvEvOBOprDZv} zr>JK|c0L94Q?1DFNDcPWyXGXqVsxLBxq%VzWwT0)LmbeSEu02kgyv%JbnU1qXH9^9 z1Q_(y9B1lSY31_#4&#;F3@`0S+M)1qlnwo#v*OiyhN~<$7*zu|jtMP^&YP-6;1l?V zf)NHzgkK2cmM39gQ4{jIYzcbm^Dasy7rPTy+8p92T=VfavAe>W6uRK-+00|Tu4A~F zUf@`8CAGy7qS(!6?pu3Ptfd>C4WHrl=W?l_K79?VK9T6?)T7|s(U_^$#Zs!7U{P$s zMLekXGVpw%!;Yu8JI5E&zDfz3;Z%^sk>0MzB#^?_|L&Y)z$c}wva;oEtlz8@?i8KTnZqbf<-zFnAKd0YeMWf4Pf%3oyXvkY7H8D;o>HvW+A)xGd_de1&x*Gy z5CH@kOJ5Z}zGq6qA>E%dj|-A;O5uuzIW*jglyc~C!S+)2XidBJeCH-f2Mo?q8OXX; zuH*|Ce?=G@bR1EJ*e>>e>y2LWL~wW1{9T!KAc!C(R=ia+e!t>^m4BLDPg96Sa#2fO z2_8n4LR1vlxCHR2@#GIPm<~tVtNe8sEoG5243hez0T(=^X8Di{U3}%K zu0xkwi?GbQL?`Bsf(aXuR!=l~CB~aM8Ow1CXH6_-;LyH6Lg$sahXSe**Nc<81u4&3 zo-a@59lu>G*FDUdPkK_`RaHSOKd5jV@8rK@TRC+c=13t^xK$f9XWQUlY;rsn^VDst zNJ%YvnPswSB4(9QY>I6GVop_MW?-a;d*|F09O9*WD|}Sv+~kJQ;Xd8nHD^2bnD0+h zf*3cC*meWNji1J@7MYEtbg^iYM&3o8>pScC>-$M_5k$@%kR)Z$FQC+dVRB>&S|$5h zzU%j@V1GJ}q6$RX8+i*$&9agyo$g*b)F32G;^FZQ*O4K2$tZb`4 zduUHY3_#{hVg4UWT+lZ*@p3NSX7wFgGWe`oA1s6K&U3zvb9}dRU1;6D&hzh$^I&va zj4D7*4Irfsklg^tYC>;1l|7>TlSPlee296CT_%l6jSmyKWZbAEdT!eH&PWewjhu2Z zsR79~!ect_AuBPT@Qih>(3dgZs{t{q!5@??!B$I<`lQs54?VL5q$Op1;mt9hyrBH?`X__~)Jg#k9vvXfOD+I=iWz{fjK_m`^J8l*l%ujN= zAuQ*WB)x!M64fg4Z$?9{F4h6=a`BsCU}UWurQ;9Y>K1Q}R0BhUvGhS5-4#bUuiLn(#x zN!^x4s@G_0wQ8&}&8D$yyJrAn7Zy8N>{IW)wHKcn5M!AfQh2rf7r1O1RqaUf68YE9 zwyO(Vo9$2DfANbv8F5L%zv5Hk0J!)KE4+`OKoLx5n%E4yWr`H>epY%l+%60trkm!R z?E*A2@W0G?`{@9B|Ba$11zcgRq=?{m8V%OXC)RC~sn=?&xL8`;Vya%`+#J6`gi%3WC8w>l-G_lOU2X_6;|w|IuM z8DddVP*G1;Zd{y%lczevX~8v4MfzRGZ;V!R169(^KKr>bC56|m#MRHN!J>=Ti=SHo z6NbsM!j@BNCQBa%Gq1#4H?GB$2UG@%v>(*zWl>kS8Ij?-3YN-nsI5kz&0gB!K9U#B z`;KG8ckarJ@?9H>c^4?s& zmUdoGDR<|P^xm~E6Xomp0d8aISyB6F#oZYAsN?&yfQ=-H~^%IA&&M8u7qJ&(KF_1$o{)) zDs!kfNZax8?@@P{NupCV;$f#)c|e@BUAud1hj#2|%W`B6*hB8TcoGF*ob5)Zs#971 z?{}e8AV12_RNLKBOSB^(LQ_qi>a&pN#7f9?%oM@TlbMxF;*Y`JzzuhUt%0GMvGt3( zh$k0uG`IBx{AE)jiSAx-r7Jw5(~t5e-bnAS@`q1}CP6p+vj1T+{S#!}cFUCh{2jaW z7V#-2(6zlQ)fnLUgoWk`@kt%M%*S4jWk%_J7|I;g^TcLGp=1S4i@9(xM$%s(p&u7^ z;=|-huhZg!dZ8wyOdKTncBc4{pRSZ!d@N!;oa;q;yMqv9<(IVc6GY_`cW}WJ1aBQU zwGP+oevhn5R*uOuK?U(cb&-1%ie<+k8qPMZt_Z=l;W^b*nXFh2$~19CGmakMkBxghbGXoART~nEy+mEtI53U(o){ z7sBnj4EWrQHoFs1Fra(>yXXb>yM%qwcDT`N13Os*W>g&r6ROOYA`9q9k6)>#K7#V6 z*luk>rrfT~(ZY9v`7DHvHTz#jr~ui|kFB5* z?I#`eZW30-Rno?0tDu&dA-Mf-KdKiqT$X4!20kA?C0pj&uWCr+vwyQQDs4&d%4($^ zo)RLQw|}Ml|6e4O%OX;_K+0`40Jw!(lfC0T0pjE)4-dSBk;7>jXS>GkF#3!<2b&=| z>;v*9_;9ktaGUs9)os?-VvEM z)$7WzQiXOIY+%NyK`Qg>&f0VHy2jBwFm@Voh2Xg;G>C?Lc`hI6yMB}|5 zF(bAQi8z~(8~!06KP+&_qX z966F#(sjqP-43WC5)OmewG=U#Y|@O+WAxQl&+@FWj^5S9Bw-y@*p@}FHKlJ-^|xfp ziO)NmOF7JyXRRwEx>bCaD>Kw`;Q9|T?1EWUgxp#25&jk3Hv{4AYafrF~ z-YtobLXy0XlIqh_ef9X#^w=$t@uky!{rKjdq&)N2_O5L^COZ6N;F4JX1B^1Sz6jxN zsA3r1Gya*maK8>eoPxI||=i+}r!c@tmf7L;Z#N-qDrp z0Ix5Cfg{!S9p6~ilOI?Q6Ly*=>F4f_dwCyM=;`5kk$hb{4babjXaBfJ(6lrk^x(>_ zpkCT&h2C81!}7|J!h=$n6&NfZl)9f~~Riu)I$BCU&@(V=@BZ?2aV0WI@8_ z_oMvdz+HL@bMQhrf_UTaVaJU(P14LY!ZAAvU;N}fIU$Qlx9}Sw)alnL!eXYJmE?6w z;e^{qNUCxIB9QivUI(25X6UEG9M`}<%x_Iz72jL}*o zE)Om(^Pc7=jn!nPM3x(xO2Bk&khag0Uexv`<6~1D3^^5Q17^SQIucrc3?CD)PR4h) z63PHKajHfK&>mQiX8yj;LCY0Br}HMd8SfaKMaPatj!Q{OV9JnQH2ghqk;L;J%yWO& zL(+#-Rs_9UCuOdidC41eq)*2${nvcCQnB(e)Q`2Sp3&nEhz}A`PUuf@+LlT@k{stz zt3x}hqprmw)0U-h9jDg<#vbMs9=1pfA7&sfr`>%b{fu;{v?9w22$5QplJ>86r!I5 z(<_g`E(2tHnU*|V?oJi^b|?vwzG~8?jGk`3addI%WSly*Lu@$i%_rj^4Y9LVn6Hd3 z=jJ{myR6oXj3MJJO>%Wj^WRL$pZ%Ll zd_3QT2=*k{X6MripoEw8Gcj-@<|R+q%QgKWmX)U z@e6r6`bYQmBqv?N$y9?eYR!jw+mMBwvD~oc^IaUnDHeCH^?P~GP(xFf;!WJICyBMJ zNob3k9$1TpbJEL0RaqYZ&ZUWW+SH5N>Ov;c7K%kK7AwRz66K$Fn7m{~?Ka*U`P$D1 z7yNe{F*7P>>0)K?PmFzpNBN!s!fqm8I1)eZX-sDX|+CC*#FP`lZ>!l2@X*#`8BmY z-1j&~1d(r3AiS6h4p$%z|3*BpY9$ZQE&sB_^+-luKgvxhoY7g4X{*xRmRknvfkI^r;4YA5;xdXwRO}>?r zogzhmj24F?MlG$@VtzY4mTuMAeTXJZ@AE%$#=H5~B6zJC$y$9A>02c_&+*fHy#P%A zn3r#@GtB8OmHb$_at2S&PCI+K;;#Kx(x55jSGoLaxImD4nv3@S+;zCIofdz(fm23W ztzdjuRqf}>x|aJ?TH+!f0bY7M$|j2vTeJ`3gfm}wFhxD+9?+;B)I`v95-PT+8uue9 zD(E;lPe%R3tE{m!_l@$TrY^6Z@v}|;Yvb;S@v~O(uM`kJf>}3g^025W|Bn`V_ot7y zm!7%5%}FC65Wn2YbGCKhFa^8?gqp3YOZ=;;%VSQNCjV1+f72?a2S+jH{t8GI*zUfF zu8m%F&-;mx4iZ7Q-RFaTqEl#n@nJ`6D(F?NAvLqYp2Ydl0@Kxa!N4^!*DAv>OZf7O zx)q|wRSvz@A}{K41MnnbUFww~aN!a1jgx*5O<27Or{=$`Z+c*J$TLpnvw%z(1M?Ya z;9c;eQ#m;cZQdpjJ$!~!A3R2rnZq=1lb0UOHrY7VGi~J>hg|{tX?H_G0pIC&P&#%> zEO>S8pSizkxR*0RA#l-_P^td0o6K&(2~KGjlEV)yMVwvmtED{|WxXZtory2y+P=Bw zl!Z-geyTtIc@rr}$+Ob?Fmzrp#^X5@-eU3+LcT{w1@A&}yh47SM%WVcKjP4SOP zEXQq5B_pfsi0DXv7k8w*C<3KY6{c1arc&5ZdLh;nZ5M?hRDz?|yvfEoe@(V%m_+c! zRcKVr>%0#{Ta-_~=@2M?o!4@sZhS$BdXyYupNBz*QeE!ezV=iK&mbdtbda}v0W5b+nZ~q!G@Sy^ zzcZ5l(Im#t)Yi7X7uv(NKETBFd(Q#Cey9ihaM}3lS?9dUl2>dE6bH{Pg1@c!!qLRF z3IDN%bQKXX^ZV$%MYPURM&oPr5M4&f%XLNyCS)`HWOMn6gVoV=DgP9T|WpY0Fppt8wl zkNyvb98>?nJACJP&OUlyo&R*jir)hjocf~*Ma~d9oXJ)?h=Xb(flB(yWVBU+ zVZ)2bK?pilbE=Xv>=Rv|C|?ml$fWfzLpLCO{3XqcVj|h%wfKT`pyYi$_a-FQeR};v z+2*$4(Z4J!Q4X~*;;n1Do(*KQ9^AnzZXSg(p< zpYyO<6*|r_ENAqP4OuB#Ny%i9^S0g%_Kdp>*%U}RU<&{GMlrv0_aU46qCJx=-4Tu3 z08@WJsH%uT0cJHIg1i$$X~WMU+rSHkYwZJ*RY^TONNb9$m=7C_@2y{S&cv0TZ3%iA ztS6*1M#x0&5t^h^6I`&Zeoc>Z1B^Z-{-`}}qd|h++Od}z9=9>K(9X5q_N8ccsaC$m ziAi7gT6ejDtM5O7W=|cRxPAs7Lu%K2_I?fSr9*A=-3JWiIhK9CxYa&WkmPtKMtU+R z#{%$%{A6!=Rax>hVfDv_jaXfesUGGWv=4Xil0=-3J`*6g;-A!;K_Y!Ru9`taY{!0< zkK5u-X9)FInw(5DF&S>xHmfO-70A$!dBmnjUNwj$!(q4_drY1Suk!&TIoL*+nrSG0 z%@;kJFAUo)C-^(USyocW8-Nkoz5GAF??$0 z;D$kxagn5V?{cuDQUpD6aT&d+V>z!b>ey~YBBYU{-Mz*Yn7pQ&U&y9!1$wQG4kM5r zS^2k~vUKDanL-wx^eo^z==PKib79pCIs55d9qTT-9K57wgwWK5DPu0n>+aXFjWjyO zG#_`j*$}UCqXFkH6DlWfGR8k08qqUcOwQ%*Vf8P9(QJn9nYJda&6=%t7;=O+#tZ>3${gB`r!r^G{66bEBy*?^$ry7$K2qxka||)^Zst(pAYFKOZ`{sl z1S%k2>qipZJJbjr6x#Rd_Z4hxCyoE-)c#-&b7nH;M9V;IDPP~9LCM^q-&_;o=XH3z z=Kx~&WqIR>jb)=Hi28xcl(Q!s-gA;86yEbkWZrXoBKKuKA$)iG8-AkVcEi7>e42Lf z^8g`RxI_t*8i$2v2+nw|h^>V!c=f!=mu4eTx+**t*zh;amB00>oE2*C+kc@k`6Y39 z@mP)K2pHl<++9b^asFYTV0rh+T4ANDl>vFs%Npk2V_IkaR-UG5OSu8|2eku#_L5_= zn22Yv$85`6KJob6(!vsFo2`&QO!A%`I4FZtYkER+rkZKYo z538DO)=|DvvCGZaY&cZ%FeW9!Ti4?U-xNBx2`qRpMYzn`q6-@D) z`E2wNx50YPTi^>=;C5V_h)NcRnRMHJQ&1Y6K9Zpm?=O3o&#Mn!S?er5B6LuJZUtvi zucuXlc@sd*uQ?EsI1~*4l#*S;9SaALNeM4#<7y(}V}PY4uVmTU+D#gHxPJkfy7zl9 zAZu%FfQ1&X+islt8y^DX`WvFeu^LZhg$;XYrh8q0LrLW6lBirT`ARTPt6Fb=SXKwa4PsXY^w#M`<`<@`$U%crpH)A zYBs3Ij0>`ZQo#Xe4%05j;~vpcPl>E*ykt z$+QKWA59!gbJ!EjitXxE2g80-ZfV2J!Y>ysUI$&r8i0zd*Z*Sgt%Bp|-flmOSsF1j zGg~ZK%(9r787*dJ28$Ui28)@QnVFeca>nof%~xL~b&`{+bCXmqx_f52rDJh0W0^=hJUKNy`khbDvM0=+3P#vNuq@K&94fdap8)Yq2QQ%X}X518P*pV<-ef&7X%PXRZ~mkXua? zj1$AD`08k-)9%#7@UXId&~ogH+&)_^nvPg9ZrUwcE;s6vGP->3g3Y4&-m!R{3-s_A zYkSS+PuXoDoQKKvv?EUA#!dmGT3Ythm3zF{9qlnn{`f}^GeSOXfWmRaj%mco`556^ ze4&WxiyZl{%-xU9>+h=9z1CJ2LdN^U!%f62w^=!ON%7(3_tad;WH~LVlXU(c1cP)L zfk)o97sO}NpL0ZZJIsZ?RZm1g*b!qViwT|D$yV>f=p@oK(Ed~K3Dd9X0pL{EGx|r(_Cn%kWeuFQb{i+ZvP?tS%p@xz)tc~(1le^oo89&n|HfqP zZyiuMET?>&vjxAIguk1wLl$IPcRqNvBtc)5jI-PrS#NuHo+sO1t0FZLU8->zaReT7O4AavbFiXNxw$NE1(s zTK?Y%zn!z4&A7{K&v->SrASS1Y_R2>fwI>V_~b3Xo|^<{@LG ze)U8U!wTxbbndX);({3Vfy@Ye;8_mh!_;%&T_XSL@D=4G6C|rT( z`tcG|jfDKL-fs2uqP#bPvEFd}#4+AH>fAAyhsX|70Ys-7e;!2nJTddL@%_Y$&5U9s zr@y<;a#5s!Eb{aJAB1Mv7fKB_VI9D^2XEqmh|E-KUK^j11BR{8?1;*TTnp~`hBj$lEjAPEd z)%6rx8#pvO@f)}XxnJ>(713r>yibZ;So~%RVweNwisX|`cg^e|7t8|o-d@DY>j2aTL*nb?OAe2nW!&3!Mr;8U2>>1q+N&(de!~=o+G=(gPzngx{eh; z@lJ|tGoMo8joSJD?NYkVL^i}5|L@e^PD)RBvcYeLP=5vA%m>Ng=FN1K3rBEn+^zKp z{;a$yBe$YrFd3tLUZw)WV=ouAYula>?uE4%&3XZTyNKPoHGQx@S`aWaA^>1&)u?#D z@!V@?&ThD0T15%%Z-o|g0oBX@hY@fqT(B~~_XK{(xu+R!?MLusmyTUcIuRO#f01qh z8C((D8eE2)IO3P$Kf{*?TCGb}P^n4!(a2(^8}fjPLrZEF9|~!M_K8`CEEg`FPM76J zf`D~hZCiQ+gn6>Ge}*u>QnY`(yY#djIpyQc%2+|$;?|9Ss@8-?`>D0lq&A2;z_r(# zOy1iIb-O2M0|qC#s&oCXX8cW^9(a+qe!E+7lg0=G-ZFt`gK+`^7yc=Ha6ISWm;VRB$b zNUs{G=q!J@qL9t`BGd-~c=3ZW*LJkTr!cxUgC%?cBCma04|{Va=ap2w6e-5*XvrGF zXeW?ibP!c!5%&_lrfy{S;>ai3C)+|gN$gWN1B^%cul>oX;NB-IGrrgCQ!fwX!}V+- z!D-I*p}PMyG9%-I2TQ%NFYVW+Z{eN|(f}fnzCubNrTxz+zKbLE|C5TZF%T87b;kF* zFC1pZwujR2SzoSM3^jAMRg$=I-Gt)##_DM@eO2>X0os-pUark;AR4W&>)#wTF z7{p}-a8eNr5bcKxXc(EXNHhoM9X1LyzJp*S8*7QMi``u##YtD#BG@3CH0Oc;1{B@D zDGQ001m#~`6U9!j5Mi>n(LBG%*D7Z05iv$<7`RxhCFh^vo%1VZZPVpSiGMqIQb>d< zPF5WV2Q|P|-~)k@cPg$+v$%^%53uIdT%<6X`77VT`>1}WO_>mgfatO=+j*M%>T`5( zi;8QgvPVHmW=m>!99Ll+ZRB2FcoPjf+L%FC{Pd3V%hqG7L2ufgjUG!@6O?izOXAaP|S4^ zfEaHredDdJJ@-^M5lph7MGRYc30ZV4xA-OIBVhopR(Ec&ZW(orDmSV%RWOz+POJC$ zZR2LcWJapGpn=KNjL7R*{pD6ZdYmAv^dXGH6b?kwNk9?72Vga^Syc@c!TU|~>EYX| zSxvMa(#@??>KKwt;kUl71zTh)n;`D~@}4K>L^zG+UiKct-byYWeaw)0UoiJlXq9D+b6 zu8YmM7ekGM1_7zq&2bDx7lugu2nniKR2`2V_GitmcTM{Ydg4B`>q2n^UVNO;mX;ZV z1SHs^EVpkko9neum^a6Z$F)$QE>VCeAFG}#z=^iGl4^zI<|&d z(6cmkP-PJFU`00jubBOZ@qC=rcuRjL>-UQj_@2vo#XWd~*bp5I3OKUYb?6MVc+=%Q zCc*~u`HHb_Xrb9akOpY_2WtKx_Sg8rzg6#~_I(-+tlvDQwKWT;0763ycCpkm%{12w zfj$d?;S(!eD}}n9PO_AS`!F9{O>HU5;;fueq|Q1h)p+a81FUC6V#+JI2p~S-4%_s^ zq3%g9w&ZS2$OvLT4L9AB?Y-x4*|Xh4uxCb zQ)`!!Vyig;wPqvy3~iv zrP$n9OfumX<~E*1bKOxRvrat*Siysd(r88Wxam)*69y8A;Cl6Wy~F=b*QQ8NvoKgl z^@Ed_$UyxpBr6F&4wpcrme3s`cuK(Di?c?c#Ezef9LA>mt2D4U;>5?I5G&28J5K%1 zL*o(+yWSzN5Zu?Oc}HVe1?n{>(s$wh{d@qhaOSwM;ej{@KHe5cZd@i;6V61zzXJ8u zXxWU599%oc;QQM85Jh!W4(aFNpCOi%4Shjl9|(N%9-eKryk@~o<55v!TAXx$IS&EF%b zxnSyL_6>Wi(_rK5B977VCJI1m4uZn-72fw`ZagS}LbFi}-To`rKT@--f>+0Jdg3Kw zehO%te8_Nh(&(5H?_BF^&z8#i=5;_S&V`51K3&slO3y}@c{*U^4;*_HiQ#}w^|!$( z$u1EIW&FT9_Dp|kbcCm;?rzff{7F;Zu(K0DuXvIWoND_aL|fz63ZxPZnyR32Bu;~k zAHca9l^pJJhN6`ZXZ@K^Dy~a_b$*Lxg)YjUT=l`Jk^8reCZGEGP*lW0=0>gH%+3ol z$Cffo$5(&GmNh=#o(7aOr|;$l_sn1JD2a6yGq9O+GykJ5Ka>MS01eGplB(u^I23QoQQ9N3@j*~TId$WD-2ML>@74~%!NoqbapLCbLz=AN>t zzqTroGo_E3aTZ?(lAJP&<@(kKHgBQ~rDkd;v||6yC@0`e3Cai+8yi~{7i+t25xD7r zy`?MHPE@3EAjY9HIk0yk@Eo||G2>>R*ifZrYiMNcDUy`j(X9K)nij1WxU|md5I8*TW z*?f`4H=M@Tw8Cx|NMDkQGaXcqR~}l%x9CJMoo;;&j7dq;a%hQ@rqUIb_p@PVn7CV2BxYgp0>%&uk3Z;{e%`#x@G(;v~CsS8T z3FY*Tr~J7m7>Uj!rY^8Q)ducDR(SWXrC0~H ziCtbIsrz)sU4YeRX}NRkvdN=)aYGs>L|WeYCeF1Mmu`sN9UL2VM*6alBu^e?7H5%5 zA2lv9%GWqv$HL4l*^1o0KHbeJkjmP1LTo}evSJoLD^5sirP|`dD-uR=-I#V>aQ!Ax zL(;v_UMvxo5@W`IXy)*CVRxhh$1d`!Y@aP1u+^x!1`UjLoA^zO_;r&TDn?DK>&q&M zT)p-cQ{G2$r}OP@XTlUE4a?7gU1Lf%Pe$067wzyOs^_FG3%gVJ@|vZy?mV^Qd=8dg}oyXJkt;xdvo0{}pmUy}Ds^AP|J6LRQN?m%o61FaIy^#+pP)WId& zGA?assVZ5@4GtSB6${r&@))VN&@vBz=~u!FHtkA1tu$Kki9C=hK<{=Qe{6VJZDHnL zHipt$6{265ZrBat!s0PhU(J*91zp-p%kq^zDvk@k zhaHh8!BP9*<9{HCdh3|2rUU`DmX@daN1xb}Jh6OU8J-w>!&^4npq1h-wd6jlN$H1b zokEW*yOjpg9=%F8S0Q}O*r)WIk&IUVqcYo5!M9&=26s>2#Ws}&u7S9zM*=8Cdo}Du zhI%4*)8~Z3cJ=ehp+4fxP06tGg@V6er3+OrAre#1{tq?evfj~hI@WU8v)+mw%r{<_ z0Hf%XNQ|EUrXlYz(jdO(V*4_KC?Zl(uIq{|M~^`)(V4AIukRptvUp)#B@r1oP;Kh} zGZY~k?)^tGo~k6bgT^}QRc(v%qtkZ}B+#&72$1M+_Q^qgRmMTKgu7CyTdB_hG7{#2 zC=ApCeLe=#H=>HZq!t9CZ6%_M#);VNW5F+{p!{)+I|s9OwDz=kaInZxhD<~v1JA0G zMAb{RsKcH<-aUCgIcf>Po(q;^{jcfrezS-bep_c+4d_ej8%!9Yn`hmsIN|*6V?*45 zp9Wgjfb371NMS1y7g_YH_ArjGyPCi=x68Zzg=cmM?}cZWpj}V+v4&!R)Nu15AturF zEz+hQKY)>D6>&%JF^?I{N(`N|;0yw$^j2S|NZJHSm%+w0Di^bQ;s7jxfNkW4EL|X; zb0Tz%W`Gj(ct&6jJ<2ezO6}+Zz z?eDgHKlV-N>oO`wz7$pXM>gvW{7sNzrZpd)`-eWf$-)8s|cwMO2lbnlQCIu>FYh?X?naGB1Kt*+OMa6MY?A(IPaqMd}*(S)>Ff+r?$kiUD_bn|1YI zUq?B$9DChvQ>W^3pyxq&;($nxp4b z>F&0;*cRC5svlRUXE3*)A55i2zIOy-Wj)X?yj{OT^fN&(vM0*?f0gh_ZDFPs_RoD% z4|DbF)@y+TcWI{GT=JUhnyAGCna8xxk)u+Q^sF$2infm20@>N2S|I7%K7lpkH z>!pY3X=BHLAWhc3gLDD3rwYW9G5I(CaVN`>cH0YU0L_O+sKE!Es0!P`h-fgcVwhLZ za&F>ny-0i+Oh?BwRwgvU*(D>8NMLl{7du85dBV)hM^QYq3^jOjK=+yl5uQ*5a-LzF+^dB|GcOAA}Z5LQ{%Z{9uUiQ@Ie z73lnHKJz~C<9bN@WqyzU5D6Yb9GWpL+&KsrIU|U~Kn41RoDv)LjIf`XG;O}+>?I`3 z&S_#=aN(6Yd*=o!X~wDmkpy|rkKPG4U-5!TAUsomj>*4B0uql312HG`_jy&$Ew55P zdl7r)+Pr>oV)D3Z2V|6-qV!0|S52%~XimJfS}cy$)`7!`O7=OgSv2;gYHRvoYIlvr zsm2N|qzBBVF(uu)@?Mesu;b&l?bJX(_sk+|vD@e7JoQAxfD8fU*_HFEiH5xix8oJ$ zq!-Ih+D!#PFhz@Q8HH>mFwC6A1!}I=e?RQpant0QU zX#)942;_FKG|T9W4DLOwV=H+nivcGUa~17X@l;n-lE?OUBKdZp#ExeB_z(UwUleVu z>k6_D?;l+}l^oyVZKEhNoIs=7X7oV1*Jd21LQ^QkNB25Q{5q{T8ZUg>o7D4>3j6*9 zc(-KUP36k5w(c@zH3AN-fD_Vm;eC@DqnTalG9uLX&P0@Ik+()A(8BH$su7Fkd;!h; zQ*JNP8m#oE;pP(qGu3poEI98^S=G3BPG$>x>TFha-#=}e3-WZ@>~OGkUDTk!n6)54 zETxDpG24GqNIplA$zCK>f@+EifoqRwnwP#$n}2yg=<`|Ljz%%al1b4_inczj2atdNEBy|>{|IBJ z5LKF*bt##jUDw8#Q*fFruQ!x}Q!Si^SUyDP@OSuUl2zP+Lys*OW5eK@1<9F6t5B!W zl&aoDoLy`t3jg|*jD>twNRB{OtLGBkkNUw3Xwjfs)&E4Ln_q0e>(Kv(rO!5L2UhUniW)Dd)VXYy z=XqF(wh`Lz;0pmgYiQVS3A#FG0PYe|bO~?zSG;&X%XCXY4FGLtob+b)H%)4OWIfQ4 zn_7)%%!);la-emUF6&kieiSZf&9q`;o;Pgv#NL_In>n0?HoaW{U!j;Ci5QJTVjv*2 zugr}?rOEfp65e>Rd9uEh+^UET^(<_@<8kCti>^GTGr3ct`0QVauIHoy3!bl zcRh#VW~qkEJC=ry)}X_QRCFXfuteF?EgB2&6WMUf2j;uK5F+8y)_Euj`|)w&ToL!y zfVTO}pQPpd_CjyE_wnQnCHN?_Bwbn#P5_jW00Q`)k~xD&?cc$}?19(zEs!`VwbqL` zaHs=TM0e{hKb!p-bXdqMM#|9o8=hf*>B0Y81o3LqAa*OAi^|aJ5^>%XMw2mh!wH}+ zyBlVIgdthtgx5MDg>w*fo)T-y_q47C1r>#^3y3IcXV~LXcnim)Yvcc*6bM`jK^ z;-lMjP}Kmk6WhZ*4~DXO9cwPxv0^IFV8)9dRn3@bTBFpEl@Q%sKm4uRsJ{sO=BDVg z-5@i-dpBBEpgdy_ZEOLAob+UOtO^%~@my0;ljMp(7Rt(>Kq{c@L>Zj=JLD&`Sl0(A zNOl*nQDTVVqm$20Iu^u4pgB0T43!t07ow}o6L4~)n~1X6Rk@A+Y@cuVpoewL{U;#p z6d=skeu9xK2hP}DebK{fyqHy)%2LnaU-m*5Bq!Qp96Y>~)~i{pO-+`!a(H)|FxEyT z(e)dIx8g##K9t}TQAS%#w0``9@q2d<}pj#3C>^B1D3EG|DW!z&yD8V~5nk}l4rLJD^p43^C5-=6IWJ4+D(5)#Pf zhgyc2L&TCNvvd4Yg&Jb9Zml6hg*rgW+RchPMRv#me~4+GVmjQ&+-E=WpNHz1=#YCX zxZkG8_!Y1wd5)rFlX*n^rs;LWEkYM`80e54Mqt;S7?0vQ;r9s6<@bi{8rr<$ddG7< z2|>uY0=4rf`olM9Zi&|ov-|-hAM-IDuep53h!|3Z0@(>@B9QO~A7b_Yb)%+7Ch#=^ zKn#W=7NqYGB&Ze&pol^s5=(+720cfGL|Ot_8nKc7ofdwn*(!K>*zVx89+&ksh9A)S zdvHNvulXAXRP-7|*kx)Smq{ib%jYE4Zaugw?Tt*h?c#8r)=N$CaBw9JOSpU3S@Gy zRiAdfj(G2zNfRzNuBfWM;F_acs8bh%2XWCy+3b7SC3}gbd#UAn$zS%;tM`7^?zI>{ zRp95e<1dZl|6Iq9Jcsmm^$Ci4<&{F z5_6ieb;ONKKQ!f>QtJ(c*y7V~E%1~ZU^JN1MpgjlC2nvL_j1Jh)95j0%PTYjy2TiT zTka6WF7E);;~1Td@LC^H#ZonQ439s}A5mmG75{iY#$_DfH9Bb|)VO`qQbMf}7&B~e zAU%j^hBT{TK=#BPehl*__^`Yu!F)?(H~^g6oyWz6!#5zR-64+4|7Yn z9BlQiRSjb7xj5GV2n@Yu0gHHBVP4XvcV8kV!TXvq;ivyQTdHV3;0)WY9jDGUo{`jj zwoS#a5PoJOz>2)C&!kb8eI@g(<~{um$5b8K!2HIVGJrrE=O^H&(Cp7;lc&!h0nL{t za==*JqY}OFstLZ>v=(wD%_M^{14VCjg3Z5a=P_pp?qsQ(2EMl=Sd~^34N0 z_^Tk3e84L=(`s6UvwqTAN|nKlDhpDy0!(S|m4#A-yr$>C{6d`q_IK9e7iGv} z1rInq&dK&@PWiovHS?v;>db|~AQ{-(lnkL=*v`^ChhVK-RXWrIHWL$ScB_ead8n?z z_T+Iv%&5AOi4gTYI3ZyxWA9Jz?dWyrXMHT%ldV%zC^Z_d8I1`XAOv|O+l!c&2NnY5 z3aG@u5Z0+x}{~+@_bxx7(itbuBrzv z^&*Y}VlEx(*pN_0U{oyDw~6iZ?i#%#ZmyK7#B;6_A(F;qQm>(;^GF_Q1LLcdy8VQV zpm@d9%PJ(-?w3smD6C3h85ske!4Sty3?OZLq2}{j%v)KSOmpjeU9w;?f>6h(ZYs-#bcCZl1rd~=bxrd9 zVp!tr424b*J!HZTa0GPmnCgIX$R_<`;$bIGNXYH*Sa+m|&tAqIYhow(yfxG3HSMVY zkrSabu9*A1AUQI*f8I)_;pe#l<~-#3Gx*XIDaot~kx)*$c~B3<*p=Obm=pp`~rH1x2sCBC0c~pR? zQFIo&aY;b z#8a;_WIB)YK%N=lJeZ-z;8|;jUep7@LJxR}hD&&t^ODdcTU!t|v4fYz7|l&!0>Zm3 zgTq=6Sz2U*GGVtx0vOw+y}2O$C?0{etm-D{jYk4_sb5XeXgp7uc<+z&ew+pbra3TE zkO9v%!+BfLEYeivi|z;KVMNIL%_fLfgzHN?eAorNhe~_6?592*d%!$j;thAg3_yeh zXOEKoO38ExEmntMEMp=A1|=B9#(RnU(f-&c?}8iK;4V;R^+1d#$=HcAoPc-n)Dd+} z$d6|Q2XUeK-dGs5Fo)~k`CnjI`%7o>MnKo7Q{j-X#2x^%QAPQ!Uf-FoY|m;%v`WvS zvR|dZ-TN2zjC%bL8mSM2KgD*Qck(3GW2}L%xd!vOFKP_exq#}pH$9uYq&TOVJx`uN z1{6U9eJ2z>Q{ulFH?=*pg7aWzWj)5v`mw3j`ccbcX~#j#x^^}rq2NsVo|Fj29#mV7 zzO0GXn|nsx^8?PM*fpeY*0D*Lq@!??w`bDXBeU-*-Uxe|fCxrWdgGS@^U6Id&KA>k z6$LLwWzT_yBgb6Lu25>svqL?H%<+Nh-F+p8$CBn(MNpmRp-cVaZ61ftIYLhM?jE(> zBT9$KG$PHJYkL78r!TTh-y;d0bl%NzAZI$dbYhAit~$!lx^joun~;wMj6$2diY?iq zcE6fy3{Jr@ptLarfA6@>ZX`oInxO74^DYq{YDq%=31h;5KZomW>lQ{W;;1?!g^2Mt z_Ed&Y(i|`#WBd6!ftk&oKHY4DnOVT;qt`e%?UPf9bFo}5!7H5L+GC{D1UT&Tu(!_n zf!j{v+SJ3h%yrwV4VLdp42^#exQF}b{pCumum8Z%{Fu^;z|c{AxoZzQv*W<8in86-?78C2nySRQxOv3am%+!g!+oCD_HHJy;xqAmio0B>KL z%?he|VtM3TF4<$CU!#A#)>1k=$NT69=;ideE$Qib<|c7Io7{aIYhqUEU-c7TE*ocI15+ykUSzzo|g4pR2 znK7^?yJdQD(nhfKa95XQ2ETG9`iYsL3}4F#;p9k4fU&`Nbr%p+?%f@HZ08I&69zcy zZ%2xK5iwG~8;SnyZD%249!kW*zRX4Ng3l<4kskx}SIxNiD#_EW_A!ZgF5~C(;engf zhj!eEcyg}x$qi=*@U=6;<+C?Ji5bz-(`pQ#Jb3Cw5)G%kj9nydFt$!{%8zTwytWTy zWV?yeF}v3W-XvmPv0I{2$)KQOf`oB>1qW`*e>XQ8uCE}~4o2ELpc$XpNAC47l6iQ? zo3~8E?!o$L3d`jBzeq0YkwHVq^FLs|)eFD?F22)jBhs7-8L|&mi z3VM~)_yd2hIjHPSR&O3gLO(Avz%UyN!7y8W9CZze`Fj*R^LO}wXE_jI$DkHN;6(&s zz@G=+_?P?-Nq=)+u{S3yg?fPvqSYJ#@Y~w}Th#F1yh`jnsrMNjUa_s}wTFx6e`?c< z4(gd`yp;e+!;-cqKDV^gaFbHcuK(mi)yOVg08p2Z%t}g7GkbKk?aR}lA%Tka6u__+ zcUS;rr(L@?f&Enb+mAX3R~639)8aOZjjMCJFwd(+&pn=i^PxmBuW z8+G`)FHcMgN8*y!&=M2Vk}AdiJcGqA4sbA&ut-X|^eeRx9kP?KT<(;3*5z|Pw{I?3 z4OqfWDt`@*^$gmua^AH9UnE56VdYz5SXZr#7--5>lEY!=;ha7{F;|coE4n4HzKPxU z-TZhP5{TXWbGOm9_33i|wB_sdbk()^=XvbSxATwF!%*ziR+s0~+R^VEMsY%)rK>;h zmA_0mCiHwe-w^oUWAOGr-h4l+yMqOC8FijXjbhVLU&ME9AO!oWhpoTYa3MNr+nhGs zib0h^Fe<({5h-#R3X&w-3A$bctw_UJf0UczEn?OmHd9m`(6847Xs5RMj*Na)BH!pA z{JjQmYO0mt4yv1o#VAR26w}vGV1JIa&J9@-(@nhRG&I|*zJ*|_X%j@*KL-Vj>g!LAzfA{R{5^^ zbD%i1cGlfw$I-G!R(7(z=z4Z5Py2`ASPy{roOtzT-HG%~b(L4|*64PT(=wwp5jsTL zV=pq33|VA1enIMOz={_gsxJJU4x1!H#ENZo0s)w;^Nwk2nDgPI93C|Lm}qxW%@LD4 zWn@vT=pYNejWb4Cbhn(z~?^Y{4(D$P+{8Vpl#erI*0NPLQY3xj0aBtMGPjRl?}F zvQ0F79Eh3$tdueCVHvl$ z+qUooksw8!gru=opQy}~E{Y4zO}mq1*1^}gOo?bRN2gmN4bB)cJW)jcW>fjBunYhD z)_wMEaC*Wh5_2v(%@7Mj;#s@y4;~5H@{iqqG=fX4#t5bp(Sw#2d=0)OE}88>X%*eG ze%=VnN&l^J!Q_M5ENq@S(+V!<=R~SqZCl0UB?Cv|Gl|_W=LbNjm7*Slei$Z=N>S@# z%i*3NCqb!9Z)EE&rD=**4NH9A03B^^`tgTVugK@Lcws#SAFE1F{TtL;jq+pVFy_(V zb1XMhlRZId$GhY)m0rR-gp$(j)q+)F?=O`#>4my;DrD@SG$k-6Tf*)*C>+pCbZYrsqsa%%& zaPGC{0j7xr!aa2J{SPkU%VEx)HiWdIZBUVi=_i4e6c9-MguDu)2Ac{Ra?vo@eA7k- zX~iHiQHj61Ft7xYduTGzV~9AkyrIDJEKVY7p=tqj?SHRm)PMGth={ZQK#+)-emnwR z%nLu!5FnG5W(wXEtR8KCo){Gwd@HwSqc-Se(U^2&)pQ}14`XvDm&Ko+9^?u zl{2`-lzaw0B_w5m_K;W-tmpf*pXf)g+_vX& zVJ?l@*STzK72WBIvY;eGNEz>VazMh*(80T#yU&K!pKs|X*SZP3kG#E}JZT0ddlbmDVfGAk`VFb~#@nx;PHo%(tfY&5jJ$js>sj)<0$>VVN-B5W$(lE(b!qC!| zS@ANlYWA|AW+IoHM3QYCMw&u)cf&oYl|+(GB*tW4k_1~VA+ zr&7aE&_pxS91lRLc~;jSPVt`#5e>F0;t3AiK>EWU-Qqe>p@7HNiPC?|%il*z7^*Dr z?)t+UKUz3vvP1J)RJ%Jg?O&0$Rb%kR8yZeE0JEB80mu98b=jV73aZG5Nz43VmtQhS zIjyV3{CXR?2RA(*ivJ&!5At7hbF$%=3QRFjBfSn}J5-L>J?w#}BGyQSOXSrcl;$t` zXo6z~Gb}Pr(D6QCms^6G8W)(9#n<*jH7Y@pj5Jdd7sID5lLnxK+GV$zl!pq+PKfWO zh-GxK7vszE>_!u%eK{h?M~;bQ?TA2&yf}(@MR_QvcH*leh;X}nDa}H|Bs(Ee?~Q~B zitc=}0Q2wlfaGvI22%}?1(7Q~REpbBev%^gleALU;thf2kfJ&}(N0c|LEm|<=HR>T z^8-Q;lt!AEjj2Ut87b_|T%D`!a_W>4u5D}|*KX@N-gji~=lsaiORbG^tC|v~P%iL$ zm`DQxRy(aY>4b>r0R;@ea|Ne_5-Y7JyBk~3FG%I)1yNJmjcz;#7N^sG;Hn@?G+k7} zmbv2HMI~b}4&s9_TQH4;e3sD=L7 z;Q@<>IIwUq`V~U+5g1?fgGeHKeDv)8dqq7&-<7ap(BGCpF{n}CCtBW-fq50W@#r`k zX)}F)g!cEB%I#E2q2h;8x$c|_DdiiPB@h{tNRP>GE_Q!p>CV`;pieN~4zsDDw&#|S za_r1r?cWJ*kkWx;r|5Tah5p-=_TBjb1}%0Ozm2>n*{shMdbK35yZrhhf1b|C+u%33 zNQm%gF}TU6+opd;U=({|>g65E{mvr}ap zs)g{HT4ABNi9!;TsncGvzqpL6BO}KS_OTo`B+Lcb%7h#oNTRg&?ogFn&=C_gl==sr zajBWNk3Tqs=(3(87?F*XrqlIynr-(gjgvoGjtX}d274^LGeFO=!G{9ZoBP6yq^y6q zm!D4w6-&k42c#4t-EoGNa&KhnOz~CVeVUN{cW^Qkyfw8M(+g&(zH=VOVfTk9m1%`KZl;qug%k>X$_?c6jBh+B@?N+wl0gOW)MF68N-1#J zZP4>s+OUwRhnpZQ3DRmRME+{m2hCa!m7h0z1O~1+7&_O0d}6oaqt!G&>X6t6_UtdH@OU7bVtPtR?T4S&YwMX7UHgJ* z7kU%7Ten|P!=leT@AE^zW5(wTx3nV~kbM2#ga1=R6wUtkA|gBaw}{LN|1Ba(L{f(H zCI1wWQzSB-1){LtuYnO|WHa`ZbicobyB;Y-RFKsIxKiRN!;Q5i4N=rk%CDvNRR)-1 z$(SQ*$P{2}$!;p>J*NP|aKFiJn5gMg1j6^`!%t?zkEc(jX;C!m$-{#|cC^WI)4!pF z9_kD zfE#piMPQY)^0{-j3SG-k5$nVk3YwW{E2|u5D_c31U@JRYDcQ;{*+c=H9g3N)JYg0| z+%MFcO9TP#7=YI>-`9Fyt`Tpf7iuN`uv!h+c>=!v`%l}-ZsW_ak^b*<`j)Nm%c>^H z6>-BBu+{jlQONRqA^{y-E5p0j_5WuFA^(3B`F}$d+34M5wU?9C?@y=(dmd$#J{IVn z^?dYd56sDEPw$}~XQvG}tzoOL&}BxF#DVq+fTTOcOz~{w;#oYB>FXsR&kRnQf#Ym< zEB%VFu;^eEDpLRZAONap#S~&O$r1&?<+D6)g!+it=m+3v2(GV_o7&+WRRo>vRapu& z-2Zqdyp(cjp399LNiRO2Cl(rB8Ss>{jqaEk*jHwqj*WKXO)|=bLsj6UQcR7~M)P;} z8w+P33Z2KD%?hSuVY$k1hA~BSfsF_v+2Jw98WxH&1PQ4DlkAByFEGkeumCZ2J}1-3 zFx!x|XsAsu9(0mb=Iz(2ye3ndeCVxRbZ+xi%@&J?R|&2zEG!0M6Ub?mTb`D#dpw>V zG>j=}@0coS+Oq|He_hN{8>`60HXck2PIVn7i0Y>}g^du2y$Z$I_RDHOrzVw0)uNy+ zTh6Ec{DrwIzYC}F6v>xX7GO1Z^xMTnagWIiJ)w8F3AT0)Bb2MKhHQW+W>{7R-Y{_8 zxLZo3 zAICA(jDz!`*_SKgVZ|GA%OO3z?3z!*=bdrzgWcgwo6CrkvwqtMDo`d>Jv?S38JnxoA? z;PRdvUam0D^o5_wfW&!g^LcQ!Q_I>?L`KdAjLOOEw5z0JWi8PLAD-ek%3%o}_e-Zz z=PVL`-^S~znAmo{*Pt^4Q%KN%wx+rjvOGnUfDURQ6|r-eNRy}ki_?_m+67v~ zRj`t7TdnF6-jT|cgYFZ!wIva;9p+Ovq5#sEHl}yX= z5%@FD!}u}`dk(ut-q-mRQ4!AA?sD&CrCFy<3o4xtyoqCI@Wrj+g#U&o*pd5^@6yr~ z_5z{F(JIb{H$lffcD(BSGz01#v{D@{mYot=U77ELi%EJoxNW^x3et>8Rps(eJdn-g z8D$`QE@>?7jxujlN=b=es&a!kw^1J`{aWIROGUF?0nQ-onQ8!Pv+Yke zp$HgpU4=}3$1@#>GkO2Y?cJ^bgs|4PAL#zgW<6=6#n%q8SYnp2)jyC|G28sjc}eG- zE_&;l60Qg35c(~z?*Hl7m=t+U#2o1;L0jD>j*@@E4H&y$tgkI2aEjBVH=R^k@Jt(QIBUf+J-a z1MoN_f7qch=}k3Q;l$Ea?HLytqe7y$ z5RhMpqJKCfzi>-F=nW&h(GfwqG_X`{f(=pGeW4ItKz&_>#G#>_3`F!5U>G|eCC`bV zaROIAEr1h9K5Av8*IBU5!4Q95dI@@oHBSbPh(2@ar7$Fs*E1P_BwFw|PNzxf-%vQ< z*1`|P7aIM0O5R4_6}ykxUSLuTju%!CI%jtUm|6bIn^ra536@aox1 zj@k&NNDU)7Bg2M>{@My1A;BqIIA9aeH*kj>vk4C+h85Yi;fY!V*+cOr5v)gPI9w-%Hp`-MtI>{D%84nNx=k|@Uw7D={k;P9ZKm3XL6}q zY`M*47<7BVw|k11d0;vI<@fuo-pCLexf*qlN$TU?NBu>#$ZD1aK%xW0(WhNKj5UO7 zZCzBqD`&(HAP(J@pJuxD7*g<|D!v$mR40^InBs)r7TWo2t6W^jhrgxhrW zT0^49pv)6QU?>kR1)XYJAS5?owLAR662wm$UlkgBHy0<@E$69}F%Oa0X#~yh-~_3P z4bft7%ogvdMVS)tGOx2=1;&476H}PF*7!>KNa^s#z;z^NyOG|=7Dq_AoBG{3y_Pv* zub$S(cayD>-s`fkz&fkw#w7=PWCDvOD;^^0mG@=;xZ-lI_CADY(6y<q^fkX#IMR{I$cbkHJA<;e9jLW3lhw-k5U7uPQopseo=LDm}8udVsF@6LY`x zb#U4|I^wL_&5 zxQdUQQG%v`{21&25^d24T>?mc>z*TvA}7-_{AE%@fCerz8^Z5zN-(L-oj(C5!pN6K z6U&3$m4FqFBQqWh!@ycv?ovpwU^rnR;*7xbqJsF`!+FQq;8v#nsGK@?oFie4sgZq^ zJAltRJ;8n=SgbScbBhffq4#B99xPO>^1C_^B!>sB>8Y-LgeLEm{Y=6Q@C6un3 zSTIS?urp9x*9=Sd0#u=xlraLv#?HBP!ZCK3k9$eUM0JfslYBSca+J};Am%RZKRtZg zYH5`?d_G>n*XJT0t*ZSa!zHYEN_DahmS$4$%Dpn_jSQzFGHY$`!hqIt?JgNcMP{@4 zd-7cmbz7bvOY+7c+7xh6zHD%X@LLA#qttdca5VigP4u0Ok8)?%jZDi6AaVyWUj`u~0R4yT}i5;whn+W_OUo}fNkp%hb z;_iy3K5Wy-(wMqGrUcxzE3~2& zXX%W7ZR`74PwSHyVCx+gmAkEY57s`0AcMv>(-$13g*%HNkzSzz53z3IZeI-$(bVQp z<~5JSofN6c5`3oEru@2rC^F-^a`zj+hMxQSN}$s@YIP(wSD~uOrG{5;XMM!%p7F>K>mWsuWFKhvU@@M#a8Sh1zbjb^1M%75SKVh^ zse=9hM`A=U&MkL_Qs2ha$yw^$uHz$|#$$XuU5=`9xmal|Cwhp49EzDt%YE$v=qYRZK%<4$IFX;PEinICQuNUR}F;cJO>w zWUH<-5@ZBJ-6)vGVZVqlczBrBYOG&5x;O%uEQ{hTZwSox0`!Nx28ua%D{C0=^b_r=banjoE!4-6gpvj? zR{mPGR|_Q!X!Y$|n#xpsWr#f850x0=ZtEYaS1z?t?i5XUj7&nJVufa{RdH)v_%2^h zzsmRSu@os1O#G z=G?y@$I#Vvr5iauh@vA-wkwLtv9?tFU98O$4z-XTDJ@Q%n}Y&5~hk!+Y*UV3U%ngx2$+s>uk*6FMBJ zrH4t|fbr8i)EElFeD31e)J%fgUTh_I0k3JErEGA@$_;qt3x4Md+6>_*9bG^f+G7`l zG}cu7xsA5 zd3=OsU5NvV+|AHcF_!o~={CeZP8{^5LPef92ENBGjgmk0O9W_*C%SwmW3;k^NF@Hw zGRKi9hgR&DI}fPE-IKRFSn^(Iv=CkNjvzk2uf z*S?*&U-#<=eR3{gHG`{|WB5xN)f<= zvy!;2WR=4?%AKE!l2wmw{g{Nvq3R#=M-$#ZYe>dyvsaYBUpa=?_xR zs!}WC2TS;cnwIQ66~F?t?4l$)r~?(XLUoDcD>yb8fs$Z=xm)|sHsSgcU6+9y8=YZ zyxIZ=K_fKnF+dX}>nY;KaNMHMBy8kfZ>il(<--Wc13_0P7Z(6pJ z$D5JrP4ci(qSu!dGdg-xck6t2tl^#E?>5mFOO(B>Ry_6+?}skeQaGMQ7tR&WiqN5* z)AGtwSYz7aPMTWGFqe(gnpmDG6xGu_m!qGkO*&_J46^xFFV|YvUCsGzQRXSmsIxxg z*p{8=>}Uj`DV>=WqJoC3S|SmZA*c!Dd*3jtmL`t$O7-# zMxiMXkmj{-b=0ur-++@iSVr1M*t^x1=;g(9x6F{u1%AsLv6|V^nwgOvQMjaAgoYGh z{eg)7nG_#&jBzNFBFBi5N+y`W5xbq`OaBx?T~GXmfinHC-e1y zAPfcbsxi*e7RlPjaVMK?@Q-^n^yR*^QryrrLnQgKf_pZ_ASVB+Mh6`MtEkv5z5{oo zg9b&^GJ)R}w(5}zece}kUueyIU?;-q_JCl5m&ix;H$SV3DG&!5+mccWf`>E?VN3Rk zY53)vK^Z;`Y~~aWYMq5WF3nvrv>^ljl)JCU_6a9@vOMwXP)A^eN}XxqB|~la7s_!% zg$~WS1DVYn;dY62AGb{!lhUh9M@w>8vVG+z#gIkI!xue})A>ckOly;-hH-Ib^{FPz zV+E(Qp+zPcSKJ7n-NFb8JK6?Z+hH`GYKwej?C+yh6Vf#VD{!7Fd6GH9BiLnBUK;pJ z1whu*mV}bbJzB};e?W`13kQ%DQ26^A?wY96Q+yrP^v1IK;LYMKbil}ECFlLVReskK zVg#Or6)>5fCL287?Dl)VJNe+>?tHrhyc54ZhTXnCtv=$qJHC$J_|^U1pFcl-yLJ2U zy}dZz@49__PH#tAy3W5p9RCIYXV)WwTisveWNs8QL?L2x6%LNyu?7dxt!h>0r&G7{hQS_tpIpbe?(Emch89bX;o}?9COB$QeltbP^?$ zt!rcfXgv`XWjKB?l0T*QSk9)C)zQs`B+|u{RnPKjDkLb$Y2asK=p+huJ4Lpc$P7hV z&c16a1}@mU$7`&>sv%|mfi`;dPuI2)2?o*o4jp41ASMELwgihO0Y}mGW|w)~C-d30%x)W4H9oHw$22pxT9BcE9yWgLw&^4g2Q z){ho}jnBPD{a(J`AHQ$g>e9u)B|{mDxX%O8X#-|7l;~t#9r!mQD0wzS57YlMAvM{B zp@JWit-M`Zj>se4ogrD09mU8#n$&{Bdy8&t1s(d3q$oYWN7MawwoBMk~GI|p%-0O#Vy=+j+($JuvyMb9$nreankTMm82s6 zX&Uf-X~I}oe<^JR<}kc#?^OPrIOp1;s%tZ`W)0GN$fhcMqKat`k>;J!o3Ef-^6+%A zA=xCZ0ly^TwM_NB@tm%qV{$s@@)jm%E|a4%QE;!hg*I_l?rXa&Q+BZ=`|XFOBoaB% z&q-mry>z=2usYV<*dunt*O2c$kX>JLf1l(SkGq^;Bdt20$@554D=f?QKj+7eqT@7! zdkH5K;6&sll`6|z%PUUg-~ndl(`#0+pN(L+FOO7l{=spFk=z9{p+m`1R&&hCyIOBE z(Tq7mMd0HgZMN zn^hpmI4PSFzHQ*CN8%tBY2nt$k)cc;jrlSnFAs?zO55esZq!^h%QQGWr=3o1p4=)k zs<%x|=}!>?_8qpnj?6Ewck^|w#D82^@q#{@B3$gaE_k{hT(!!z@h03v~$degQTp?cFGjmjH4SP4GFUdr>nft&--9p>n?IKgsvxWi{cj5 z;4p^0*@fuIeuohg6>-wse&lBMX-pV3OpDHAoa0!^3Zf^a8Y&X8NKQ&Mz^yk)SrtsE zMlv=F22^>tieTJPaFV}2g5`Hwzubo9HSkYV_kP@Go^92pnX z!gW1+FnLdkcL8a$!XW|%bkPd#dR|D%sj{Lu_<@enn?ml^S87;@&jNuP_!z)x1xgQy zsBBWL^(YLo(jg_+-v}uWq=HMJKQBYo1N~4^;g-aIoVAzVN$>87rF91`8iTYe(Vc^# z%`STbM?sU!&OxBbR_BP`xnQ};kjfZ^qGb{|46JC2tW@=|kVDl7o7oJipTu${GAwHY7G4bHH3wg5?Pug|_9daXE|iq;X;IhM}OTd%Pyw zjpYYNrH%6=1N3-}14~V2m=zaGk5THoHAJn2t21DDP*~#6>A4y=6n&8~&5F_`YmgrU zNJU3QkIy73N|P0JWzTP*B+6-7{%IF{qI{WYMo|NlymAvIs*Q`LgZE>dPRn=ZdEe3{ z#Ol-HSFl0^?^IXPA;eg}3Am3D6Gj5XR&4Qe6h;PmZpY(qg^Fb-4$zdtcK;>S{laqw zkaDdI!q_I75Ct6L!C#egeHhgt7mZq5_o7@O)nhJJ;%7k3_31+7h-E`0bjbn$d~qg92Fu%YSlZX&A9KH0O=(>}{<%L!ynp zO6VvRpAQqnSwN2%OTFyH@RSni@`Lvw#N$H?ufT>4BY-8;R=mM(P?GXJM=+pIPv4o4 z7ylBLQ_j4A)vjUB(%;fe>vOrZY8QA(?=Q|t7j&1ym=qHWN zt0Z3b)A15VO<2zk!o)9JHQ7bEsdCUI)CW;G3!ntya015>v~ZlC*dlPNulMx_@ejyE z*XPVhS#t+wKLwnp-P_IP)GBa}%)M-MHDBSjz&~fOf?vgOcH#2@pN+=7$>eT^3Ha*J z5c1FuquBTEjG{7jiOk|LM7Z_^alCkvU=!Lge@KWoLxc~p8FtRy8Dv!*Z2H06UbXml zyd~2gY;MB>9)H`6zL(nO2ptV?WzB>QHX~aTF9IXHs<_cI$|HF^r3{~Ls2d$Trr;C5 zE!%VQR^}6rIJW?b@UaAs@aYR2ArwL2G=T>E2Czh7);^vZj^P7*mzg?$Uu$tZ_cSg; z(I77`=$%jR|16Mz9|iBs9uA-Ut!@LHs)0|Npg|s9&?qk#*yG6!@H5?BEfZXS>UF;% zy)|a40Imns8s{PKE2XeAE2DmPb)Pw8(tWkFu-Eq^%)xa0o|pSOJOOeA=W8kCR`XYI zos(jdSR*39`>X}F$TK$;p5igPSr-AQ!l&)nFGZ{)-}u@uHXGeI zl@EVtWCK?UV!Kra6YnvXfWE8N`LEsZ*atL%oxqd#vkB<~>3Wt*PSN~f51qFKU^t#V zTB>LIyhapz2StA1lbE)u04A=wCJlMA85dN6`l3 z#4H;-zva*ykoE(iNOj$d_9gN8|H)SR_=|{^xxVOxty;lVKEki5 z1EMXIg|;HSDYGew*6yUe^(1}@I)<#qR46xdNSuEJ7S2IBRjB{^aLkx z3U?$KTdJufq@nA40Az;8IFPhe@)1MYbFcaY0K;WA>hZjuoz04==Elj?&Z}8K@Y>c6 z$J5PqxRjs_(8t`N>uIV;kf#*xz|GNpyh!ZduysUGm@;w7chdWTIcv}#*Om?y49mBc zb!`mO=l{&HO)HzRvIf_6kR%$=yyqZh7Xczo=6^5V!U{@D%#)brUmBwNKvF*65J%c^5#GO+*iY?rk7&tP$gWi2$tg2%GVQ z7uaaV3kI+t2lCZ}FzyHP!#<^8JXcu51?kru2lAJHTJKUD%?Q_CrC{IFaD0I8I7V?Z z4%VD)?Lc!UndPT@G_yy&y_lW88KDYA`BY63we#;Il1-)zozj3DmY%P{ znWV}pGeVq!Qg6Ujk=@XS7wg(z21??dv(I1hg^`pKunR)%jC_NEG?KJl9C`S8OGUgt)L;sH*VJMgTAywtEIY(VGQB$Yg@m4F&KYiP0 zI!D1=*jVac7-{liX!&Wyg?0k{RLmcI7dYEpo_wm2c~^s~y_FAZ*^s_++exekt*%U? zw&PgW*G-_x3C9*}mm5*^#m$y>K0S6=Nkh1kb}#3YS)8N&bQ09tPt2gC^u(aQAFF~h zau_-ooT{QGHrnARUH%6``Uh2bg#E#pN-D(5GhQIbo06mPo6!ErD4Y7VJ+-d6XH%|v z3KE-IKQ#(Bsr3bCe0@WX72sx=z|a_pjm*%P)?Q;57m4kWM`C3qpE|L=f!toh+tj@; zt-gV-Zzy}(xk?z(jH{WQ8M(*VS44S~xqsLN!7C02+6m)du7eUV zHklzCwdokUE=d!0T+(DEqc=8gk@sf??a+dj5!hMFo5sg2*q~CEx5ugaM$Aos*I)(_ z^Gyf}`-$AtzGHPoNK3Qq*nTP1vZ%04@EHA2VAXlewG}+XC%k_WYPr;!z!f;8eQLMRVcdIma59#- zkLO51&JaxPki5!>fE&!Z6NzS*f;)ngGLbTW{~fc~>*;S1@pb4*0_17&Brnau$GZ5pc?XJse&IV^&z1;GJ@l#6V zpTbgApt!e<^{>K$>5O6ne1L{VC`2WpxA{Xh)(&J73O*YLh!Ahi7 zvtMw8wb+8PdAGmuiAZgd^+AYn_(V;2k-4}r?JgX`!{B6eNQE0reKuLIxX<(mdZM1i)WjkGt<(TT(@D?t@=R6 zQ~!``4DUza5s5;eH39~xaez4@y)2RZ+c<&NfGq;8P$PsPtDSn3z!%H1Q~Xw{l>q^^ zE>Z%KsZz1^FbKE`sCZ;@Cx}vUH5jE5ndsPzzGMQ^1y-?yX5*nm!p~GwXnnxfjx2WG z5}9J0!o)a?Tj7_I&2r^f2a~C{H;a@klj^^+oRKWG)hc$2YhThBcQg)VEfDl)Ei(d+ z>?nG!DkzTU5g2hjYOre~Nj}7{p^xYw(0PmiO|f7+sS$+noMZ=R9@s$s69!-*g$ve! zF`h&S!hfV-+}B$n1nD&%08y{X`Q0)|(W3$jLZVwX;9>$sLeK%~eSqrW|!VZuR_qe(6(W-6y0V~$13}UkepR(#T~E7COb$_j$UogU7gv}LXha% z+XPMKQoSwH@`jUG`qgDCWg1d((#5XU!3)(<;p}uocmjGlI_jz-Hg!r3V z;8!`E;*>>S2O}tWw;C+4!w@A8TU)^Szb@>zKVO) zq49QSrz*)%Pxk}?U@uDc=0L$ivq?6!Tf1uQGxwIfbyTF*)uHMXu2L&2P5^1+UIyI6 z3HA+b-q2V#IEFLtR4t5X%TWRTsm5F# z{zll-lzc~?!UuSk3cxBR38?(_6&DeK_|U2a7Orh83HzNt0QNPW#wi zNScP?qm{|6y_0Z={CpFxCMUFx{oPnRHM~Ln`KQ*LE^ybfv|&?cp1e~8?c)E-9=S=;db_TiOZDZ=%!j_Eu4K#`>It^306IUYq` z4eb0~&gQ`1Fyr}Fw0q`Ez(~zbgLs{tv`&LI!RElVUk#wEJbwGsTQz<4?T;x({J(EA z=e(U@VN8moiIhZTE6Xp_cl^ihB~Y8v1S-oH2~-z~Be7noD+)t(gH#lwnsF)$m;P?3 zltf|M+ST<$VTp4J7;QZ+`%&1m%Ko#jTU*FD0Z1DiR>S{~($@6(zoZQ|@Bbog&8V;x z5J^W;`_5>Aj`J=y;7sGC6&N57fvC1&s>si1B6D=DuD!*v4_T+KdHqN#si&nXL*e1L zAe8>Zzzih*y!4M~f;@z~4~gx7PJD+2OTL7J`wNf{Io#ZsWj6<7NZ4;R*q~dX3Lf?r z;;Lh=Fi3Cvim}-b)yxOusE_iWJk89;B|sl?7+pXPBFZ0XS5X;Hr#cIB)?jn z4-cB~fEc^qa{truZU2GhYu;G8d~d|{Up65BN&@-yfSO3-r(dk7P(|&mRtr`ymXU|K z+-s+9;N?u?o3{UIi*KL3y5}&A;vD(c{uWoq1Y~T)N-7av=P`0bP;&lb0!m+n?1yIl z$4Gh}uk_zt?kFUE#$az{`LaTl)-rHNmXWkTVQ2!D*j#I;2e_8GkW%$?4^>J-W7ZC# z2Ian?TwqA-^)@g_Y%iF9c5g^*OW(s#w+Ic5r-OoHVB3iR&wLLXR1YQhhQvN6vYvR6 zY2iZolN#jNYA5>J(;wgQd}|tD@&S@L7H3J}VfNMmZ+IGMUwB5|WF^SYN_$ z|Fak-mL2=w-X6^Poh)C~<($o{j}-u)UCdp@tkyWRr#>E0&wapDsBLWL%*eS|qaM*A zGV#!)2X=~Mi>}rjDWI|Y zqr!V>!9D^4Cw%M=-oys)xiHwB(N#HBYP(l9wA(4lPb#~+bYro;A=G_^Ae7xG^rq7Q z1`th1^!GqK1afymA>DYP1q|fZu)pe9@@64lTG_M8RB!zaa{|Lp{CP&Sx;79-J-QoJ z%AP0y$^j?>WUe2S5Ker2#Xz_t1x|cEc}8frP*JXfY8=N*!98VpqwJ7Hafu|7`kS5W zu;qlcdb@V(!Z9P9zmZJ;T7iTbf^$7shy@`nHH9`L9R#qA{3#AQMQ`Kt`Z351l5I3L zi*$M*3Waoo3sR8rewRo%{^sy0ZdpYI6uuz-pKgrfv}C6XzR&CVI-8Wh;A%sp8PM`lf(WHc35BdFt=Ic^`fSq296<@grte2XAD z!%}Z8ZigOD>p;n}CNYG81|YF?_*8>HvWkW@gl{soeq)iNCBwhOBFsxt-P(XQZey3+ zB3?Nezools1PrSv@1~K)xapJ{!{5f@;V#LfJpWqDp8pZNk0Pn6!YsI)QVbxQ_P}f* zuWhYNa6UJ4PWKv9;VG|u2VS6)djA>wq9+(nqzdaV!Mk4UG(a8J;T0>$4J&@gGb@|h*=tUMSOH7x zUr1UEnT=h6+G!yohdz%+*@p6gt7zdal88&n>bGFL?SdbmZ(@=p5=aXRQ7kS%dTAj# zv!oV9Rr_#z5$w9j1_+VK6GY4C`zVBypz z{B8nl$2Jd#Kh^Q9qroz;0_1hBFGBo*!@B4?qagqivGmMPU1X!crO|;@=^nAT8FSjIzID1t&)j&?IZPmQx*$Zx2v!?E&%iMYi3d~nm|Dp=>J zQ+=OQ7dVA)e9&)IZ?)BNMOcd9UM}~3Ygb?9k~1~kt6MVF>M%iAC&H9_k-6Zc1Z+0M zVAZg}@iwtn%NWoeV@+*)%d(BWDhcMCJJhP@%U)5g*Z^?a!JVwwx`#?|e6%R1yWw+b z1&#*?TwUdSQPq#CM)R}O=PZcUQ2^rCRpW%<8b$-1&EdPeYQioRvEvWvavK#RnRJnG zKbY_u=%@gg@W5z&(*#xNX6K`4$?~A+7sIy-&Pq>gF;c;gd zCD?ns9x>_B&J&;!rP9AkB3u_NY5~xovpKN7x%v~3c3$f9^42=;V^i`uUEqK^h!4`Z z$H;ko#CC`70AP0?jb)5|UUC(Ijh=JUp?jtfha=hVQs}k>8vRD8%+bw(45pEK7xut{ zT8ccg?LZ#6vpw{=tpN&suoUI30S0TPsG1w!n47u+WkwrKvml$_4|7FyHPg+Hxkpm&6w1GoG9-px=BFG{-CNi9{943q{H)jf zL37cvtVlfHF14RgAqjptVyJlj+A!=BwV4zXkwXSRey02Q$R}ae_F9h+Yq}#hAy!MM z!sNH%{KMpu!?aPqV(4rB#M6a5C^IW_`HB_S2mCO|TlIH(2@}#i<`4~WHi^D75PJZ(%7VsOblO#FJHo(ULZYqDPyIcm!;i77X5(P?GS|qM@H&W z>v6)RJv{g8)%+cf9uK>B#qgGn2nI1iEImW}H)g?+r5BqvSjNGT*5S6RYPiktpADH> z$MpW#V-yFgsVZI4ZTn)j=9|3D44Qb38G#a`JTAa$Q$WM&sm}th#1rfJZ zB5AogQlDv)+k`>}gaWJ_eEyqZ5c#-aehkZCZN<+u^5gi6X{N^}h>y~b$xg?2*C<++ z+DDZ;yu!Tfdz#z}E(jON7vW+1*gUM9{2rgAT~Xo0x$-9bXs43ytLskk?c{-bmS5DW zrqe9C1;4KYunYKJ9x*Eo5scq8Q`9MhL0Pk7%!V5o0?>D0>?@vqo$b z1MOWnUkh$gYYwJoI(Dt_l7L8>&XH{*A1HYn z1mW2W@|qG4w*7x&8J3vP$A4UL7kXofGP>7@NA0>2>uLVROw5t|?uKVwaJ$@I=gJqO zb7$EjT{UE!!`?*Cnqpg7SWu?IlK?CNZ#2Ys-F6#VA4uF0BG2wwaj1A zZG7(u9CHSm%3-6|sA;u_6CEGGgn<1NcU z^Y=Zd0rL(pMy&34D@H70zngkf^yTlxRCZ?LKy(ziJq$c0SGf$8w;d5ETLC6D-*M1d z82eE9y3zvSE>}j3H815e({CPKb1;zg>>2~?B;xo$oP#{>!`a)-Q!rIE*!TGKCAF5|d}>zYG9MLm)1izg%(jyj&` zs`tGT5Sl;lywG*x*5NxZ)UvLnP^=Z>dm@c_+7wPXm8rr}*W3*Z%j049=$9Q0QS_dr zUQgRQ))ellKBeCOfRRH({t64TqWK-A}803$iuPnSZr zLi@2bOfWtZ!9?@b{04iXeYtwAV;(|#}m zZoDS=Rd#8=1HLaeQXNe|O1pdGX$&_9%-FC9RPNp3Fcdc)`%_iAwqs;!yzzi+*#aj# z_9I5;g&8=!c;u(>c*5P7#^-)z64>*@i~QL=Bov~n2s*E5+XNsnkA}WbeJ9NV7@JN6 z(QwdxCy8{me~j z&d;KME9^gCUnnJYwzALa2Ojc(U#N>Tw$UOG9%BL)>EGp^BNS`yWQ8F1!~;=8?9b3{ z!jHMBXW>M6jQq#MeZ=!#rS*roOmtSb6L|EgbW((R>|x3}&IXcvsKSwD**hqeM)hw| z62zu4lBpyd@e?T{2t7%e7lPDpXda4;APpp`45-|#)5a$v( z0=1AXZ@JpplxB9u5s+*jcti^G%==tt5FAR@4|+R&Y-Jz)Fu%9y26L|o%aTc&t&P6a z<_F#B{ve3lcm@%%y>L+^VmA^z5Tgm-2nS2Rb`$b$bNINAltE6T_?Pv2`dyv<#$2fN zWcKm(t@*RNZKWzqP5br<$={_F?ajcc4Mg82V1R@N7%AxnNkXPi`i9FGsj!^VK{eTY zkN&OpkxGGhV*;zE;`=ob%q}sp7fMo7wkfRQ#5_@%O~8o7wEu_&J>dSI6*yu6Pf(kU zX2yorR;VeH9>K9g@!TxcrTMG?8$0oQ*k=`Cn2KQ*A?krff(8nN6cq>>A=(=_P?#@h zr05zrz;KOcw9R4}VYuZ#z_2fPFoa9sNDL-odk{DhGaWDz1JvKa*QRKcjvN8_w?OKr zVVvORPsTWcm4Nv*FvJ5A5#s?z!UhJ7gqAKxnJg8Ky_MeqO|#(R(?4bMhYut!D4q>3QX9J?cv!UQ*8iodN=jn8ae z&^I%^bJ&mR=UFVVK*99S^nW!9-eIDgFz!Uc?NQkzd}0VBZxyUTF{4#e-uy`T-x&_b zWL71P3&0_d6-r-FO9>xM>j*&P=f)V6{J8e01srW# z{pUS0|H>T&Ha^iZ|HK;bCMK`|kTxMJG^!2HLp;O9ir<0zOmbt9&CO|2o(!V=NWSi_ zw)sCj?mzRcw>};LeC5~Y{%^mprq5x0KolteyX5!v{`v$&lJez!cyqqq{Pz7iwi9Lj zZSnbfw`q5?>vlkJz5R9P*O2z^FXp@(7tL!aw~HUqg4z^w0i{;#*fU|(6tdj=8>bkp zW2BO(DzX}@1sB(t^8(D#Qp{L<0(FR~oJRgR^yG&8`j(V?N+tz`7;Z6iri$=~jkSTp zHN({8ab^JybG&w_!x#e{=0Js!W5wS&#e-#{l?ZIF3J*jn+=o)bQq2!bAt4U5%!Oak zl|1+vI%On~nXLN%FgV$b%yhe{Lod*EG@OHE!OA_j#li!VjDS!`f>j%Ox`@%P(ycUp zVCqh46!g-eh9g;{jRVK+ZvU|v+o!U6^oA@$C?6zFJx6#oB;shmsoQ+-9ej}jWDl=_ z5aZv(%qdyj` zOB}zR{6KZt@(}uL-#)*+cG>5pqLX!c1RS9=E$ zp+>9GXxsxQ&}uLrTqht}gF~oE2Ps62Qi~1?O05q*j8wxpJ16%|J?BK(*%-+@*W+2k zVdV-g2VVFw>I2`q|KEwCa$N!O-wV8L*&0miPDV{%gj3~pBgI@kxqs{iB?}qc*t2ah zPYM%C44`TSwyNZFQuw!pW(D-g2p|?`3(QaNjPhGM$%j{vOR45|x~sgD=HzD`^RZcP z66w@IEKwGoBYWRXR{6a;1KiLrln0g$y}xo{h#yi6nnvL)M{A{&?XP+UteS0(cLs=e zn|eDQ8PapXx#vS}*HHMiQ~D^7=n1i7fXkHtT+XNeGyZvVxe|XAR%-dOOGsnB$Y?3c zQ21bOUo{-zyTxcfe~jnkytbT&`S605=&ygWL?tG*k5Tk;qhGZd*_+;hc4yMh36Ij9 zTD%&?!t+JvnHOlwWul1pJrKl0IhYs-VqtCv>Q*v|V_{IjiG^`6MF7_^mPY{B9ZVVn zIhZ`PnmCx&1l%vN;DSygi1!t6FwwFxG0dQZ4lWB>E3#m7Beks5mK{vEqPJ$GJl7 zNGk4l%rcMC;uhE5LnKMCu3-le_k#9C-6;t zF1Ibwf6U2=`=@6uz|{zzR@t$1P<}sik)}+e`JdEKi$#Yd0YnP+im`Kh+^r4Catakc zZ;H@`!(yLKh=1k19qS1*2$>>L^gf-Mr0Qu=C+x55K~lZ3tHWq|UVNE(W&ll!unK(XWt#?Dm5yM3a2x1tWRS<@rM$ zE0}!ag(C6X8N7`w*mq4zk_?07OTYV6L$LeATcg!?z5egNtK{=XLGU(ro`sSD>~x@p z{9R3c#c)SOv;Fm7U9&?8Z)91Ms7fmx8H~M!53P@sFX+XCo;eq&2IK9-Yc+%vG!o4Zd>BFWpg#A2%;z+orPhi3G=|!BX@Yp ztzE}&$ryhaJ52Q=35FN!P$K&seBjLV?>|BKw& zD<6rFDDN>u8Ix;Y3Jfgw&i4e0j<5Nr#6785UFn9H3@HRp0cxA$nzZ?Zl$LbrkIKKL z&7c_?7Cx&DD73F55JHjJ(2)DQ@9>|(hLxe(kSO1b-V19jM-=?E@vUy%(*R_enD75E z_g1lO#?Sg^n3@YM84Kp(*4Rac1ZkTc1p5HnDJ9+U9OMMF0Kud%g>?U5(v((5XwXheA((7?h7|@eXP3_KrNH#^ zd|6W1p=oz%a0e0zNc4D4kV6O(V|hhjuGVU8hP|{+v#$!YjMlVh+@(d721=S|Wz*$n zkJRA#AY5Hd3(W*gb5(3_J_F;m)yq=!@A&;cUUTI4$<6q$7A?0HS`*+jAHHhMNoeT%IL!8C?3xYxq5n=*OLWLs5v1C2YJsOfj=nrIdNIWspTZ7_}rm;gX#Fp(hNkuksCzY6$ zQ4ISqQdZ%|1Ua*Z-y1-@=W%ksBx7-q)u6ISgV_u~lF<+BZ(Qj>rpW&THUp6T7jis! zt>EQ#s;ma{y8L#&wq}ciPs*~rm5>eIieKsgjtjby&7Q4zqjxgY6iA`f6h7h;{R`8& zu;!$DP8lzm25-LyV9lkPJu@52vAyA-8MTM?n|OhasQ`}Ei~Ml7m{cA8PbI45&7*bB z=*Lgc`f^ePUgv%;33Rrlo(2-DUtmwj*>=bxQ*t@}Q1|d;KXDvfo4$91$QJbnx`@z^*<;|rCUG8odp=+zb~=*P|E%| zWSisczWvvzzdfJ>6Hf=4{3ODogCnymeBCEiYoVdZ7sGU`;>_0vbTugYomqZ{TO{p#s5&44JR$Kxo}CD8vE`oD30e; z8dzn*LRjJ_F0)-mA^-5C4A*LMygYXJq^%b>c6pY{pOjf z9@X{(Ui_suC~Wh>sPZq24ODtxh(VQ0wXy^aOg3ZZmG2766onLEx>IvlD84_?i*|d~IYw!YpsYDAi z*Tr=&g`23ZC`Jx*tuhnlcD2V?Vt7B#>ks0VYA%qEWp@R>jA1iRm7IpBj8elW1}_wj zEEuWin#0$@O1yJ^?q$w;#$$o~K=2z?ZOb&dJ#J|4y#ivO+uquhN^Ac!@2v1x{**p6y6IIK5o*vvy6arMZD9R8~b?;8(P$CV)zo!Ns<_ z$wArY`Qbn|&V4UcQZXp1iCY8yez_c5vY#a$6;(S4&j3=5qGW8hX(Ykl6v>cZjLMli zP&W2$>LdSCWki)Nk8OVj+YF;}dz?q;-sJFob!y;(ggLw-Vu6dM0k=%+@SS#C$Mfqqm1b zsC4p_chDp;vZ+YzsZ9Z%7%8kdP&5;?Y@Dy17HY3l&r~p9oG=9vRUmzdZhWwIeQ3Wq z_yzf5_*IiPO#b3m6g!^V2dgOS&Z`dHEsCllqgrr>&I>{Nw)y}&{SaF3dXy-h3aSUb z^|_$N9M%dmgKVC?BHd;HrDS?nm}sCP0;KVz0akMf&)qJIieJf98=D%VRU403xljnnvNLmj_B>l6lX&;%tSVX#bo_%4n8W zxmWQ$5Cow!B#OXyDbv>9&gw-HZQiTta7~=I^5DBaDYXA{{TFSzgwc`x->&~7>#|S$ zO`6VUeoe&`=8vp26JDLhinRjO*~-W5kS?cfrYNKT2ADTbgwQ*MIQhl{7g8z4Z_PHs z;uEQ`Az>kDEjgV23q$ScRPe%sxcoPf46~(>n66HX!%o+s^&~l3ZbT4*t(@BPXM_M} zYY9yXA(lGp+IYs|g__vp)K^c(=0#3Z?(t0>tx=}$f(<0d%Nq3&ZKrH)Ag#uby@iXVh{baQ4 zQ0o-XPCd**8h9EsrK`?ZtS3^&PJh&8=vLA%gkXbv$yjv4)9xt!ay`}kFH>g7lCOV$ zYc6$~FT7=ez%40WsS7)AMbEy*9Iy*FGuA)7I$S`9mE8Pk8$l4cRmJgMTUGZ|f30&u9=S$j%FiV!{yTMmY{1AU1-5)Ojy5a- zNqR7`?9vNnouWX#Bwmw`x-RxJByC(=Y~R^9U$zYa4%2wj{xNu8MdT3zBpXp0`DI+` zF^#~kp!89T@ye(Mw%aOWxGy`<@tGgls0?{HwnwJ=_yVtGlvBY$LOIv8j?8Oz-0$YI zk_*z5zSyDXng*Wd3`xIGmsQpO;9FTv`WA=+OUVj4Ba_GQ&rqi8lnle zOtJAPsjFjmiR%Y`4DxpGp$wnzeh9tEr9=D6JCrf4?e@E{RB0@OkzZ^4<+l(BPb37Y z;kF&zMgtfyJraX8p(YnwYcYq+Wn{tgnDRRU&x31HGPSLwHF_j6^(`c5?iu}ngQP=& zZgz(E#hk6_OrPWs8Ti>kZ?vr4vXl<7lFDsItd&&WE&f?*24ZdzmB96oaal$pnLq~S zZ4wou*PD$YurnsHCUf#M&=yyp?wZRYc4E z1pk9@3GPcUm2^}-89tT1hX0r!(HRdU`LGPZ_I|oAg&OECndPg7e6)^(!$wN^vsS| z&hYRdmB!b;ry)uGYXFKBD&3Yq%Ks4e?1}Ac zer^-fZd&filTu@wc;T8l%1HQf(A1*I%}=9!o^N^3d0y}cUhIg{aLEr5D)m&ET$VCtV;4Zy~M@&Y!3ibEbmq~T@v8=MerRX6NKrM zhwC2W4q#ND$-s=b4W1ECUFn|+Z$)KWy9ndpc%z!1h*rFs;CVD)=`RAEiAUKF|M#;-T~7u_XY2OBC-vZ zv(DYUB4qS=Naw;b5?8eVD3%yeTpO7;sxHodG;+is#|l^?@v0Xwq3;`xmoOTVK%>o( zFkqacN{qnN^v)s0rGB<_W0{3=aqyZOI;R(BJa(I&Ro;O^Sz(O~m=%T19R2FEw*=sN zHYwuG&@7ph=*anX*BH(=iI+mn8DV)GZT6e~So6h+Y*#1cV23CT&WggE#Xw?cSaxWJ zjx)b-;ZtFYl9{~WAF;yoEb8Y%@M*S!_V_cfp_I|j(}iZgzWxyl`aw#z-CR^tXtxc< zoy;{LHxIeh_w|7-SKj517zQ}`!U3_>DutZKA2Ba7-#SBxBL6y>5{um(k^DqV+m7elz^?D(}cy8`m|@^S=+YWqj0Q%E7RY^tD&e(`U)D(QVbkjT)bj>@#Q z(M}p?6?A?oT}Tst zc3alq&JTUBt$@2NAwyYU4``0DM#DT`XP6qCsJ&nVX_MjQMfOYTnWyEq7*vknF3^p4 zWyMYmuO3nUL}bQnvxmj0zow9v-Zbr40&!21yZ78hP&F^stT$t+QnzEjEndE7gBg$6 zP4ISMYaz#BD{@f#Q_J?ajD{G!-@I{gTch^;vyz)OCB^;CEAn29Mp&q~sj&PD#%VKx#*>QVtUTtsge5JMq#PEl zO*bnFNA554InDUBi4j}tA{OjEV7RO9e2E{Ls?&azQ6H@>PHprvO#GVMc0^)7#?HmVfu1k^FA!6>I z5>c4bZkNi{<;cbtPOU{&_9d7k)J`CgC_xm!*?}8NaLd;tWlZIC2ie-iC}@DTWxRXj ze#WpTHzJBkg2@k!)PS&rFlEQz99$~g5c6?AVJ>78MIW~N_7$^poSv#i=a@0b5h?yM zEH<$Tg~EDpecE|)34C%dC5|R>{u;Ngei-LyM^D_qHsz{@W%2Nyh3hV^SP;@|Th6>Ybp_?9i)QK_D)muxQC39Vc4IBl-(6%$ma z1QC|fHLsx((3i*&7E64C_1TC#!`z4(fx3@%gz2n3kPPoCaqDI9c-Y0?&~*K>m7++C z$Sp@e_nG>>vI0M+$_%MiiqU7ILiO~7u(zt*Nn8ZV6G@SWwKVBy-zdhDuSg;PqA~E; zYjwBXQo(Df^3h?MpD+yLUaR{ZwwpYdI66lWfnDy(GX=_`!JlImCF;R)$eYzXX4wPs z+kE~u@)EB<)$F|QN{UT~`6n%#tjqVl+85FI_(ZJf&o5pyh4hcz&*K?=dprB*@js;= z)aycv9B^{C?Bt2Mb!*f*?mZ($8K5)1Jo?#X#+P%qe{o%$QnA4(}%iS<3|xfe6Lh7J2ZGZ z)OEIQ1i9!wnW>6xe`I7g@klow11NacpQ0dVP?As2?#6ytE@lYO9`ecH-w-Iqtr32k z+d<|mq3e{Gd0|q7xiK2u)SO#ifxy4xdA7(oboyCZAkq)kKlr$qh}{jjk!~D`JewT1lFkNe)3GijLUq^OlbpGEJ-mPbqO!1n%TgHfW$MKFS z`k5W_EhTZtW4o`5U84n~`kJosJ4@3R$v+irS>jg|RL7{kwsGZ_5U^1}An^@=cT>zC z{kdqX%`vBCABOyLmey9S?-aWHETIBgo9ST}YQze?|Gg<{BXZCW-=LzK$0wOtc;nle z3nn7#Mt*-6p&u~quJ|FLipV8tDe@{IrUR{YpBQy?=qCQf@aIp&dFsGr3*-`w?NOp3 zVW!8#tY-fa!HYHGvr@$NxQD4D%z9#K&QsXv%C|Wz=NWA*jk*AN>U1Y7Z zKcnO(4-Ljy5H#NxDLT>#Kljix8s zECs_}kIu9ZxVX5+XISNvI{n*|0Y{eYxI7=F#!gSG`{gWNaoYFA3sQ@uTC}Ik<1I&g zx47+WG;wBPTFtW1CuS%+lHUo9nmZUo??#ryL4S+K9cZ~+w~l959WjG<21>UMEEbZb zc3nb=mQvM%L0Vv|MrP)o+1ny?dD!B7f9gA#yvI87#z!Tlbl2*~%Yy-GH`Lh!0%|>q`AqAjJsZ* z9<;svN!qQ3%i`lM4~C5<5@2&;=?D@LNhE!v<)O&_TQq=4JR9lccbF-UlED84 zLa&AF&G7x2Lf!kJbwY}KdP;Iv!%r1Qd*|j1jnCyf$1A=Y*}pd41)mO{b8vchb#>!G ztwz)M_=T;D;QK4kQ7G-u>&4RGd4I)|nQ~|j!rlCQ(O>3yxam+GO18(ZIx<7}DHT#2V5}?|}gBRQ7i%Tb`zG z6SEYZPt2LhwkZCA1wZks-#SV~AL<;7n>KwDJU8!fS6Qxe;SMFwmC7!SPjV#?E=;n? zL|Lh_sTcrMRGJh_L`+Nwg3}Kdq@E-Z%fc%HaJByPiC+hrzgB!#SyNCLP%v?UOi-iR z39txtrJN6t2w`PMlog7A&qf4b4G%^X@{*l@Z~X z2LoiG;SdCeLq1?(-U}!q*m~c82KGU|qUXJ-iq21AILFGE-J)!uqO79EBB2_#98n{p zOvSZI1eG4E_^>GbfJ5y>eY}itdf8Ha6*~)_p~hcoUD>Idk9iR;$irkaA-HxJv z^Lp1*>%6Dorv5V5W*GOvNQ6mW2-FV4=Fxtg8=-BDT$kLCBz@$8LO0g+_+ef}9jpf^ zuWeo*?>8L#6$kFQRM{6nW$j4X?WHj#tem4ids2RR>+O04uH$*JFv9*OSZXrL-+1HL zkvM-YxOQ-SxK*P2vN^C#(AeCBTP6!@HPZ{>|MQ+mKqPI^Cxv=MjMKvS53dY|l$ARs zFJUmJX$4O7Wqon^F<{AS={L=2OZxZLF%e?}tJ1OE#xEhsEdm=CaJ5q23)x785q+1` z9ne}@1|lk7|6G1r$umObK23BLsGf6*`ksS~#tGw)wauIssHW0o&o9=-fTPlcjeGOm zm7K@_{Qw29*v)xGpdx8))<%nD9Q+5KbTvV!7&!YW*Y*hsZBs`$>iBa5Od;wH`BphA zu?hR~i9$PCYda*>GfBnRAsbc2K?CArn4E17m)1&GOxa@tV((-j8x~upTfm>=*7as0 zhRvuaHnf6Oa5Fb$aJJ9JWKyZbX5JLt42E&#C=P*BxT0?YOK7V=bWHpv7$$q;vyq$L zY7}|wM*^?<+K|j>gC$!ZI4%_bsc#u)Z3HXQDru)+zESbq&5%@k0^n48(ctiW=1$pS zDAuloWKs{DXa$I1W^TWo+1mmY~%(aex1Zzs3JkzgUpq zXE8;ZF`FuD-zBGjJM446PktgU1<*1Mw_rd*-@rX7W3tv6LK%k!F|Srtot2$O@KH`> zWjXFT;ElI!lprjir6_VcAGiG`XXW|h_n^@qef!|e60nN7#%1ZuG5SXvSUQ?*_}sJ* zw!ateR9Xm?jhr4fb^2q}MtK3kd<0LulvCMAX9VTrz(DqZg`(|maf}8z@|VmDM3_=<)Kf#!Q@j0BW>|o$C_o9Mjw`LrkV`} zkh}r|UC{rzU;vV}5QD6%szjMmifrThxp-gvLH`s470)x+Xp9b7>lHm&D;vbb(`XD< zf0ioJjXCH9z_h7L=D!XDVt&H{VmuM(xQB4*xM>QO(^#<%(Fw(!n?c8kSM}78Zg6Cj z_?InYln5b#b3*}p4XR{(_Na7WeEi(4L1t-?Mgl+Z4tj8PY%Q4eO{+whJ;AG^@?Vyu zJt4D)#!P!`=^0J>c4Nd4`dh&EJz;`l8Kp!lY)A%B&|V=ub(|VfBd!s2mlU9gXQwPI zMVL@K7|_=Xbz+WA346!AJ58U_yEQQqOwdY%an!~iUj=Quf?rTs`yX|+x4{=fX*PC+@}Z=_kzGr8wY?>KTL$N^jb85{i*k$C*@GM^#LL4tPQ6tqwDcx%GmEIE*1CVZfeu#X_Mvo<4M@~?Wx?q z`r|6Iy6gFR478*By_%Gz`-%4BWb*^f-k0O`bki6#{q|Hs3k03G`3TLv-oW4PGmvp- ztLV#NFIbu8uCvu4O~wRafy%t78%234CNf&oy2%y`4S$Z%=0tho&E4_%#U@#kMT_*e zGeyGvFSTQTJKHRV;5m!g!zB{9a1X&?J;d&=c~s$#u~D{K5|u>n0#&mZs8emQ#vj)U zX<9|+$>i=}^XCt{N>y$SxH4hUi2w@Es-ITN+LEwYf~c(KJ+N~`osBg_jl%?6V&)Hi zv0`VLVfNE+p#TmeGR1jkz6I8;N|l6I7$cfzbVD$7GN$0*s1*!NsGV6ntouW1d9|;T z_Nxv!4AI}}?s$w+rpW?W`%I`^KkoeKDbnYkpwYR}W3#M&Sy?>3wT00^`V!puVC=q% zo~^a%fMr7p+A6tAW2zfBw?SBC@767-$f)L5XZ+AO7xp7={QVe=*1TE{o#xfAIAbun zbHI^Q6R}ScvY$;bCEmS4*TT-8rVG>}LMX%*hL{D&GSXCfpvwGx#Fde#2eTd&713j} z9(;G2#lWdGm<5>oz6Dlm`oKdpbZK0BVZ?Du#l-T>qn~i>cMVTpMPHR^UOMNAuvlPs zK(axLU9H(wWyp?Vj$P-}``FMD+&x{SMGDK)LqVZR&k4>7V2MlUMMSYpg~2kbpVZTPs}Ym?%x zIcar<0@jVUD`=-E@3?YXn3QdPwJ{@1KMxGpHrOBT4N&eh&UD;!re%X`y7nTa?oUv1ju!sKaORQt9rozPd+jD-)yof9JE# zHsaYUWuDu^o*c*v3Iw<1ZVLs4cv(fs$@G(Vl>(d)EQVATnXaj!ro4>x)Lg;Zhm^E8Gn{v zOhc!KANm!Tptc3i#VEs6l|Q9Bj4pzi9txK58_U|kDjlt{nE;^W?D`>v0Qp^Xg+mL& zM3dT)1>5)<;ovgnlg@&z3uvY1qme?b#Uf|5Ci6lOkb^{vm?sJ=kU_Vm40WkD=EopF z>}{;XOy&>Fj}s|6o&GA%LADnHarBBp5%>;eQ0-IH$gbD5UYcu@QQ7jH^>`=!TY<%u zQ4_^-AE7C=>3&>(+Xq?EQ;K#js3_k$FMMRts`RM*D6=3`*q$<)i=uua}l2&0nIj_E-mOQ8V1Y!Lv&7Lnn55|O+J zATF74i#isjdeRRXzc^wgks*Cu&9qG$+CIo`p|{NLw_&qz5! zMWIE_)eXp@o5C<^QFN~xuyn6N9$Or-qJ@{s9g^65lP%O2U$bQDs2kydez?&T@e;Q7*8Et4oJ;`FSzV(eplJ;z{i2t8!_ zJpP&YErV%9x^9Nr%;c>la%KW9&skb|C|!D)6v2&$0=_iUIbyhQVjcQ$L(KGO)WZGl zs25@j%+y98&R>SDf`a>I*t>uxcPWU=lRZYl047!+K!8=gQ4v+(g@Pr4fG5>E|2nOw zGEeVc>;)bXU->Wa#cA1Q8-z*>sSLzERswsiS!&rBC0TU4zD|61`T&prBevWB#Q`jj zwVGUXUcq2_ve@aV+8@+IWE-hKO7^Mahqp6vO(A}v}Me5A|4t%bLoX3B|o}x6k9mxO~d_=>Zd=)iAxn00dt<};Dds+}F zrv@#v=oCP*{d-cC&-9@2J>+J~kN^QX%t8~WDXXREUN$|~`@AEBnxnU+UF=j6s1vq= zR(#IIgGZ>c_R=b6Hsmrc}%|c4fg4sXkucB5=*bo+==p zw@pf&*ms!nm7WeV^C4}J;Ak%GFcI$%)$DNS@W1>3;#H z5c>k$dJyyo5ov`(R>=$ZHwG<}D(Ur6q{!`y{ zsImDv^2Y3BXeewV{!DwPr~Oc2^JOT$06)~;N(UV42Y|3RHs9e)(EH#6wmeP*3UPqr z91uFEQcCJh@OhSh-&kl|3;3#ZnxA9L9bVf%Lt~lceW?AQ8TW0V0fx zMLcCc{N6~K)#BcW(|`1UQhKPAmmSOM9e$?go0@iin-FUi-1nr2JY;nmA5RMDp^TL8 zxD2lge@e1}ngk%40@nuUU?n-|(P9+G2_v9wIQ-KVquPsd;ru8GA^KO=Pyx`i`*Wxx z)#;6Z4aq_3c|bBXshp#clnR|1MszYHWl;~NjaIylfn-j7EM))OLLMKG+;E}jHJ z_X7k-=r3PH{t7)jGR)r_Z(7hviTdJ8Z(OwM^&V?hL4p3{8Nykfa|=gqu_hg`1Ae#=~@Oq*rvpuT9)X!N!OdEG{?j!$&XRGvGK|Ob#&Qaw>)-oV+ zWK+C!-1T~?*pi8%TDN%jFX58_dKmu-L&(7QRI>J^MV(p~R zWTjbc-HFYAELa9kR3Og+v*D^&pQPzGV(G*wfa8z%CX}Id+E_)VYO_#ymsNtnj_#a^ zr^?<5RpGcV>K;_c>-lxe=72cMF3J&&?7VuA-oq|Lz{<8G(HXiH8GxnN>qIo^q z0z$mF@X1I#)YkrQr~TQi1K3@|0&V=fY2QPLsXsj~lDRPv{+iEn2pJBDP<&C)f*H8B z(!3kJ)3T_MIg8k2zz2=w5@xB(u6vu?e2EW`k?-2m%bz`ji8!IpE-UxQ@7kf&w6;WswF=Ar zL-c>D5bt{U(K4!XU`57>MqWv=QZi7U3*pNvh(Q4u{hy-4$>AC#`J{s#S3XKuk_K1!SPzmqk+^5Q_J!Qq zVWG6S15OAErs2zM^(_#nx&|aJDN+v+WAYQThs9ELEEW(xDFHY&=|;MMAJGaCQ#+At z8)@;9jJLWO;t7MaqqLc_S!UUT)ZSWm`Uzqs+rpzd=0*?!9gBh4+b4R}PN^OJj=(pBDFRB7J5;25S z;(H2jgHc%mL*?z3Tb$?WftcqCXoKP1D69fUAv0bv zDi6x@X)?I@*bPKFzCo7s-i{-5AI&6AO3)k29* zf>8dcD{AJ9?K}B77A#|g2`XZ|SjJU+EaYeBf=UA*^MywZuLW^7yA}vm&fw`?`Uuo! z4R-`J(KJXOx4qd0fx2@YT8$d1a+4l9?rThHP-PaqWkLq9Jl}8_5$;%W0OXB?Ze@gd zCdlmv*u!VGNRl-`2djAHPAUcL5ha?LF zPTqr)4)l!zyOv;r+7oxWbsBdX-^+patf9hjvh_I^sA_JfTvrTlh2$;n>o-imu!~l! z3%|1$2?ehijT$HoV*u}5)T=;97LzSHCmtVNMxZSnf-Vmgm@@m3HC#`#LDIFBFyz;a zrHoEhl_1tEO>N{VFKaAKfozB-N?U_~O(#}_X}WDJ|EMqRJjWb9&q+8b8(p*9SbiUP zGR*mx-lVygFQ9173zN`y(FxLi`d1+|y4P$mvEabJHyiL+e!?>qpeNA($uIgYl&N2h zGJ&wri(5<)Z>+)q);m0&d6Yk$u$8}wK0)rw%}4s1En7o0{1f?~|0D9cIh=|G{}K6` zr$|I>Q&*0ra?D4w8Z&R8Wsu5PB1NGuufFdd$U3aDEpua!))r55Gz>JMn_J>m(Y!6O zEb8na2rXI8Y1|h+vaPyVv3fdMYQuHF^BtLj$lRr=lh1RYkrdp)T@8fLl{&Pfb?=K( zg^*>d)WNSjZZV^-+L4b1B>nCgJdQUOg$_-~_C`z0!0d1hlxIxM#pw}f<~xuNil*m< zMEGUpayu$6%fuIGF+&&77T1h0Lj$SZt!lt}u$t@;pC78p5J@;(#YOZd1Q$7Be zolTZNNNI!hvwAH3>`snbsb27MrEEO&aM!2DNp?tZOJ+F#3bI##Q z&#NDYe^AK_9xs5P)*++UQI{moIFAUO$Itt=B(57|cLalKsBMRBpy*Ep6ANenv{O?s zA-^eiFa;eMmOGu})}^S<`Y?y1w8ntQs-c7gjS{0DOq6g0G+&MT?-4027eD_6I+J}u z`Ucf#@8M&jS18Rhf-+WGI=>nen=b$%Cy0h2EO3vMjJiN16U2nPO}yfGyUX`m@M6M>dSoBS?Hs4tyv!|aN+9@ERJK#A_xZK&@LYE3*-7MliAT*5m#vd+ zm*K?)1Y|WD5ycSalY!=n?YHmlkj*mqjC41UwB`#29e?f!Du{S{i9(sS@9c)4c^)wL zCEUdQ&Y&Ahfvj3VJKr%uEFB`6F#H74631j!zX4R@Kx9R1%)W69!!mXpD?WOMT#RElA0X`O!SmY# zvp$uO#0dKn1^E}fg8=?=TMR_M;}X;h8otd<7??jJ*8Wm07qfEWdQBYR})?dT>{!TLaCb<8&!zR|>B){X0_1vUYs?2VG#a>aNymtv$rdVft`3%-H zZnu}ONGd%Yf`dFdNfrj?%D2NJ#VI-RB|jBE)Ver?nk zfZg(ys0^dmQ$6~}-5uBCR4!*3&JIC7rJMFx0OMM!9Zv1rXYNh3Jj7f=$HESC7_wxf zK#~9akSw$qiWTM@cu~P45|n6oO}(_770Ohs^8t26{qk@n!8`=7QRIy@+TM;-Bg8;E zezlb2gn}eIly!?w5T;Te-p2M2NNqHefTVJb^P8V!xA||^(dcH&kBRP{WO%+n=V_Gn zGQ^Hh`fLKiM#YpymZ=d{3c!QCdnf%f6Ob1ksHON?(Q2`_cEBJPL-$0YfP2OJqogWK6ljIg!Lu)uK6x4F34 z&6ntoK9|f8nN)OKSr)XkqW*!R_bf})D@sU%uJZZFgyShX?p3c=inIbty8dK7J>o&V zwda*~$6F*5`PkxMs9m&f65t{_dP5XT^Ye10)l zLKyaEXnASv#-nSc`NoUCa%~=D0yPx=hbrL{VKCRpPhI2MmvX#8&-R10agDex@Rv`8QKpVmiGu z!bfNayc8^2Y9x*yR)HPAXqFADXc^{JnRIU2xT2a3a9!Y++o20WtHQeoI-ci{1!^CE zLGxtcN6OI+#yyW|G~WaCHY3k%BnJGly8}BCe$_a!oPD+Sr}j>R6#DADymI6EoAhTv z3dSP~Ij6@ATYc`~cif%nSsA+0d>lQ60<9m7K#qnYq3e}jf!8;KCz&XUdlN&^&o|tZ z-!RZS2qP52P`Aw5Kf!K3g>tH8b>nvmTm#0|p-{V))YA1!X>J#De7D`p4mu%Yfa^D= zb1!xPL<)7xoT3gBHoO<%GOf(+hJ2(83!oW}u7d5;Jmx6I;SSZWQJK`ZL^?`ZXMjvw zMb-eOd@^XM9S`jlZt=tvpH-Mr8PN|c-Or=fZa~YE!qt&rd((WMAL15gR05i^_Z;xR z+jRa~O4T)G8q)d915xG)vmy*)6TPtqEwYFgr6s$ zKKG=rUC&qm_1?w4-7iBJ8rVkIRwnof>E?d}`MKIq%S;FDl5xm?MT@YVt_`(Vc{QDH z2A9!rnW6#rEw+BF4NV%SQN@gBrrPJ3i4kpz&&^r`qmTOa$6VpdFToiIS4g6o@ink3 zMad?or0+T!V+qVzJ=iHy$kbf3Y|OXK$X7#!h4Jq8xH1-QU?BJ{frwb2>U3p;YVucG zvFN{komf+!PZCc+LqC;O$4Uy(Kz&p97<-Ftiy3jAfhf78hkGUS8S6W!>GGuL%=D4| z)>LxL8sqT)>A}>uuV&aU*80jtY-SGOz#tRe)(a0!0g93IXj@Ovw6x9A zNyC4@Y1xLl-w(Ev(`PA9xv?#q%EtHP_9bJYgzyi{xp(H8#_oce6D9V3%8MxFzJ_&w z(#Uzh6n}zmpEr7X&&Jh6F3)1e(ey*oCmrYuKY{*sn@Oa#x?_u8DOpI#l7X&duH8A! zX!1p?(5Xx=8$(~pWp```xCpYiyki)eWQ@+WayFKdB-z_eZdOA_Z=dF9*^4&pO=PdH z{VIJMCrck&JQg*NuhEJa7f343(g`J|6khqpQe==E38t+L4*sEafnW z2uS?WagKYTd`a;8DSkH)cNt(Kh&}n*GB1H&oHxPRmZXRO$*|>`X z@YZqCzZ%;_TdSwwgC_~U;?!$kpVP!A*zJ2osGl8!tN2Jq=m!YpPo>!(H^ki@Ja>r| zGFObd5`Z>e+N>yfT56``KQE4W7{|gX9X3x`Mr+G(jq2E{Z><1rbQc=zL>;_v`P5*V zA8<5!Mqcl#L+XZD8e{n5uD9=0?NDC3uX(tM2f&gZn5CZ&=S$OI2N+5IMwGW5VOI@* zChN%@!Z|)8v0%RaJ_~||t{LA)Ta&c}9Cc~~UTKk^R=dnC-X~iDT0V~cEx&{VR2NVC zF{d{1d7h-tX|(u@o2o8u9!zwwGA`b4PIkQca$MQ`uO&xJo5GhfAUG>v>~Uw z@@In9ugpnX+Qx0gFo2ssgSocI2mWj?cz68GP#!gkIR!u}xLmV~jnG-m)D9m=Yr%K8 zs+E}S!!gJ9cTgxYPI6iwQuPhDga=HS$!t(JXjYQ&Z96!&j0tcgo}`psOY2MfGz&bM zs-U=Pt?ePWQHI7QP4FL|3VmcR3B-0)izKzyZ+1^b3)S(e9ZU3N9lx&E;5sKhwQKO0 zAdWt;ja6nLu|#uL{3>^-bS0(*@XL5sNxkV_6^={JWyD3t)iCp5S2rZ^L}{%JoKi8` zUHmo_sA~3)^m!ZX4&$e7mfm>WF0*hpR}{I*+iLb7tv;7T=Sda)T`wILx;@%WeAonM z2Tv82oPkpv+L4J^O7>C9P|(<*171D(F*}Lj^zdMB=`kp|cJEk2wfqW2D3m+Wy)Tz} znVoDUraXCF!@#kJaYR)m7Ot1A3tYW-dvu^SOhH3Wa6ZjW|+x+gC1$MREkLk#f zY5k}jsBFTGy-j(xsqRCHSs!fX>l)7uTP0PUle;UQ8D5knjlvG}ECHn79;T%`@@Tnt ztK{z|Ao?&N+xe4O<=(5eP9I$0=L6kf)eezz7=gj`*C2qe#EH}>!8n8- zu?a=9YdZ5URQr9Q>2DV07>pO;oNY-h8@JeR!)(uz4t6hqM>qOjH*(q2mZ`c>EVU5$+e|a+c<^bWEzNjR%r7~9$VPog>%`5M{?kby=x||w z>g;`CNbM+!t?)84`5>y?tcZDet1PPgeGI(2!N|82R_%T0<=BDn&H4N4Q4~Mjd3lPZ zexs4IU*1AxpV?xPVUTac;?0fLYb8v#7@ueh7w35U4`aLoA;017U>y;yRc@VQ(zBhp z)2I)gkHZsxBcoUC_oF#BlwXIxM#xOV~B(eGP`)J4I=Y6W47vd|%6 zOZT-vgDLFdOASmm{D(M!mk1;+#y8e}l%kKoTmEQKShfQ+8x{G@u59aFbO93f4w&CV zQ0aUyy3k05$lK)0EqMe;jOonlGaUH?v?#RvKobmYqcP|4xn1)00P2QS*TXV7objFq zV6wj{HYW5>QVaN0wV33P<($0CDJ;!$$q2K z?jOu$yBblx{#>>jipulQ#u!I5N!GnZ&ppL789p_pSy;-M8FnVw+x56%F8AEGFsGF= zDxYdoZc$r6q4iN``!fg~Ttl-H(oR?H)jS1DbMUpPH|B*zj?`p^GgXd2_i@1$7309y z9l%;>bY>GUy)tQ9rQ;3vXi*Qh+!=Fne>Guw;)_oN?Bd`3znDAAs5rKDU55}LK|*kM z2=4Cg?(XjHu0b0K?(Xg$+}+(Bg1g?1thM(U z8a9lL18O$yV!MLUG+HAhNmn|mpdP zG0{p+o@i6Ryxnf>jFZ_&ZIK?}&^>ozMY4Edl*y1$%OiXN&?-~Yto=s6$5h>_pizAK zc)&1bs~@7DD|GbgL{&ks;5S@z<~;4fjL7DL)LZuD>pU8#r%V<|y;k#H#}@hA zmGnkoTFe9KPrM^HpX_#+{@2SN-dAT~6|c{`32!d07us*nqX};>*>4$dSF;&E9`7D; zemoqNQ2%)DcRqW2ytqv9zN~nCft{RV?*8%mvdg`Z;_ZeHJc4_9yLB=i6n0BZ#L|wP z`!WZCSvo}t#fDx0BIQbh!`&5Ct);5YJbn_O@s)%o!;`m?FM=s|r#cLwW^aE5r#5DgX$cB;!wY#wCNWgObe>>Pof8lA3y zJo6cAT7seShSd+VrZgqan#^;CM_j2+H%$TV=bkJ*)byA7sN6hfmn*3^sY8^M)f*2y z78nwT&Fq9y?xNQlv@NP=9Wyq>^)QJWRIyGr5BAK~A54#=*o(I74n zNc!34nZyuWe8LsRXNC8?*Ax4(_E`8;Y$`ay;z(Xyxch45TT%4xEMzYRDfEiq74E-; ziLqMgn`s45fL^2C_GgA-$FRB+`@*Nq?)8o4)oHLg>6Yg5VIJwqP_aaJogy)jzpE5} zHgtacO7LZXTjYXQ>>KrRg6cJ@cQw-|mXQLJ`}DBeu_a-17?1*+RnAuM}7?lI;!I3{GvznpwgozX-He9+~k}e!7er-qU*<9jkTvcm(RTL z?QZfjHG@;wRHzq6MiWs(Bry2*>RIgTCqYxH4XOl3tqGvE%7P zcN!%>)#~@@D4gov^F9KR6?5xJO1&0#las000|!Xn8NUCmyEliLKNZiH4D?D3zWiot zKq`1_q#Y9!?i!`>W;v1gAYe}ESf@48%m($EQ1Z_Hha5@we^ zTL>c^12+dsL$$)8I@kv23#w+ACr{ty?iq11bPbA;^GU`x{P-% z0G;2GHNC&P?d;imWS8tDu=n)k$woBOtUw@tXeqaVFD)pVL?hecg>dUcj9p*MPD@oe z!GGv|bf-0_G&pi+o!^!iU}J^4G?JJ`6J+s|z%asg(ssX`6!tzLH%?%mJTJK)TK!Qw z9|>=+1Zwnl^+WqPbR&tER+fTCQ^AgFU(ZXn@Uj8?C|?i!+FIG$({{9nxGzkseiw?$ zela zZO^Oo6QsM3i!uN?ofTh|k)Rwl5%E;(R*l-N-2B(H!AzJ)qh?j#nls#C~u)*d0cx zcs<|RMIrAbQI}tw7P#+aV|HItSEvCFdhnYx!yoC;C>|Yks`5JaQ{>O;-K6(E5k7sB zv9)Ype%J<%c;IIEHR2&@x8rAHVgr)=@ei+P!)D;R=Fhs6X^sg4M`2H*8Ye{{o~wJz z)n?{ZNqV1}Wm+57o=M#c=xcnC9^3uT;9py}O5UdAuJ0sn^>nJJgVvR~VZ$EX?xZ6t z4k5ZYXH$(0zPZp~B%fukve3}az} z=M&DnUSDR{GWrGLXeIS_j65i&BRTGCzp^`(anXfD3@e@}dCSvUx@7qlZG95$r21YJ zw4ZaahP{f>O!=fF8kW&r!z4&Of4fvW1C*HB~3ZwE218BcB zP~K^F{vsL9W+-?TEOe=|j}0n&N%e&&vfCG!(%zXThoZFk#73TF=~I_4FItRv@5`py zPKKs<%qb~aAwypCDM?wbi6Mh_wa@UDXGRSvK`->rw}i=DN!wavML8m zqc&KTBx|RakB`%+IflI4)UQyr)enO?XB8n2kIqk0ojnp9lPE_iBi*&k%`xPeAFdb2 z=s#Df0*T{<9mS9oek>iSP|uSbIxIVPYnneP(1F#CK^r0}R3);uS6TqQ;ILUlxZwEm zJjDl9!ms6<*fP18P|~mbxnC5529{k3OlWoAW?qE0G3V(!nQ_f?@*xWwrp$Qm`$(~I zK;i_p;^^tr#2~U}{{eo|gO3EQ?9h@vy9bxT9nyYMv$TURjEX$X4boThiD2erbt7`j zBj3JC*Oj_?$`D1noBrqq^vgLFd_j*}?+6MnQ>9=oQMd+0A#|m@7)AU~{dn$L4YQJ3 z{&NX8@}yN?BJM? zn8^c84R<7hj=TZN!wo9^M#Oym*KH*a6hE#mChd3Ryb)h2P#|AFT{_}2LrM-p57)_d z9C?WNRA!+S!XE3UNbcxfl^kk3D%_*!nzvz`Pc?1`%j zC17k%xZxTj8M8oQ4HT)nuJHMjUTM;r1sXrQi9_l zMXrraS%$VdA*vyC8te*j!qyFX-y`+F&lV&txZh@sxN92^>}uih1E}e~x;3|^urvr( zJ8=KE$Z`Udi4Ygy!T225Q$pMmb|>=)nj{V$PMfKbu$m-PnFtrCV-Er@95cmjEvrJesunJUdYN zDEh*z(8A?ex}j*Sa`~fG`dma`G)aO0IMVG$aF<%~O+L$^E#{hwu;}Qw%pHPX)u-KQC?lW6hi$()L@NqWQ*xusX`6FetJvzhX6K*}k0*z@f;F`@xl*90 z=%kE0P;b$(Blj_Ij{8<|iGKdjidjH0Ut*7kw1cts?a6OoBs1LP^j(29Wjp z#wRoQ%J%ZOJFEdReQQyAF8d4e!I z&u3}?%G&9vv`o{payhT9(uK4>QJ=esdW8LKj>wVA-dCf5m4?0Xo3W1I2t7_+}{m`MIrV+|+ zwdIfT?`ZiPBK^ZmFvVp&f@TDi&$j!mQ`(gzw1)71C_)zvT1_~>vN;=KMt)P`dL-q~ z*o+Iog%^E8Gr=@JrbhZs%sX{Ea{(2T3vVbeXv{kZfY>^gx1<3IlFXDPwly1$J-u~q zi`{l+{gX6#x)T)>d}lr)yq%L>5eYPOk51%lmbRaYE6(d46Q97FXv>0H{aDFQ+a$`i zuP4oNKTjp&3;C*4hu$?W6qA&Gn=$2`3u|qWaxt>AOr%qReWPR!TOs-CKoAEqTZAqm zvAQ_g0u>N(7gtjboV&MDy6pY083%}O*C#QW80n}HuTv_eRJuI%xg%PwDFNAevrk;# zZ8kO>i|pslNhpnY7%5yRKrw8}>IcwlrTw0Pdu7YRcM5wjHG4CX6)CCO3~=n^cE)SC#sj z1xQSC(5lZbqH#5Mp6S^6NY0%0K>ZBNvb%)&4PpZ+|_8k)ampRo`@H347zF$C$M17O~w{ zw)UkaFUgVeR(d%Tu9Ussy)g$l;e$3v3BLA4h{bWme2^sHmJfIx^ zz-Q)Pe->|8ZdejJ&;VtmLxnUJuMqD7`>y$iMvRjlmb}&~k4Z_RaQx`O6W^XE)2Wp;lkZU>|1+ z%f03>TRT5K9-u&}2Zow~*)gfj7lf0EEuuK>7pZ10tsQ@+ed=1h@1DRvg_;}%&E zmsdw$;Qje)O{YOZW?y@urs*=tAt0Ss&?Y(i(7BdTqXN`{Kxl zLUJSMV2{r0E^M!}FoO9Bn%?l9I0e1}ItOa}8sc!%$Da_sSL&DWSa)Old}IU62Ek8$ zBG~oa4Sncw_xiy*9>?K`RAP<8O#|wIkj2um%hl>8rQRP7UR7wfk08-FlnW5;FVVCepb%#JPJu^aeCs#ZnlnTZCWO-1=U?5o zr(tInC9M9|00F7lUE3ZPx`Y<}o*A;gq9d9V|L?W-v7%i#2k(QrY}Rtx$y6(5AWQTe z+Wlb%6Iv7MU9D_{wQdF$i1;3Inp7$?_Jyi`vxtz~o{4?{y^8!p$zr2`$G$-B0J~$Q zqR4}rVkDgAnf?BY6_usDo?r+ut>8?OhG6E>R{4K&bH7Cm@etNyD`iQsd?xq}L{oS5 z&s`1&*b%C0ETkSa??oZUV0i zVTX&bgu7#dcz^IgwSnS;feAw~Yjq8{T;NTbhKb9xQQO|o-oe=EtHHQ}0UaCnB(*Z4 zKpQp@^H5YUJw*|yOU3)7f70=*%%5~@eEfe)$33b{76xUMt$U`?6SAH~6jwn>g5}Fe z-6*uLilwHQ0Er_mu{s`X+*~zW1t>zjAmA5N^4a%UYPg$;r$xvE{Fn+*JrJsxn=B4y zErdHAQltFLJfr;1I4nNO%hCs7NOGc7UgAi zKYKPb4k8^0LBepQFWb8MAkIfEMsJ}90Pzk)7g1ZH-Z;ehaK!nK47rT}e1cxbXOOTh zww_?2!SuX|rkFAkd!h=~DQR$L809GEMo1-!QmXEYB8o(^Z7w4ai6nC!s!ts_^39c!P?!83J{wDHrbQ|7nhn^eyoG0(`%5iv4XbT8!y`-y8Vz1WvG zmuaq>4cbd4@DvS5)Oul>`H)K7Or=C@wTG3|pWGfsj1%PpZYaVP zpPqD!I_EYTi%Ld9IGmi)djvd=;m1hg+y}&)p2FgCO z5Brj}^Ji>YZ_o|=xA))ddTLJft`@K&7v?|?W;npv7~@$nr;9G!EI8V%Q9@QfY!O3F z=GB9*A`Pl^Ui;H4OwBmfksV{XBO~m2RFrm}!K_ zl=pWzLB98ZD=Te6zRqrl1w0?5?wgs$>PcG)Ohq+uO-CJi+%&Pk1{OJRxuCg0HSO7h z0g!gY10DxfPSf*HdY?k$) zkef=lYe2XA0vl_8A ztmxX;j-*&4lhe7@(|_kvxXy-uaq+hztnx@Dw9S69W$|hz{+td+lmSNMZev6Bxbm?w z_vvxLP6_Pcj? z>BB*-Xr)=mVcSASg4HB;3nU=(Z_ zTi?q9Hi?3>6DV-I^Oen3?^qRct|SYgirxl4?SJZJ-D4S{taluT6o(0l8>uX?6-bb_ zJvXBkIvjz%l+)2TgxVfC$FGTwB}Ty*whTVO38eNfHDK1CX)zT?#VBH1n{T)m(DzBj z~)^yy*H>-``Z4lvseLFnR@cmT6)9B~PU z>MnitkZ>km$ult94n=a)ly3r*S{;fI0V>$yL>j-wgi^isDhxVdEK8AVK@iPPm(n@~>0HSgyFRQIyIyK8@#BZ09(dpzO9)S;hBFy>pbG^)*oIw5mQCi$&~-rU{|KB`Oivu*_9s%q>O#_(dX*XU zf+!^mGgR+_m5-iStKybdxv%JPDx;hM15764$uBXEnVc=;Ix zeLbvJR{xIKpn}_yMGSAaYPC8Dy(KU)&QLcLa!XCw7HWE4Pfr1oaX>~$m2D1=biEWJ z+aTM3!L1)u0W9TnIL6_=gHG^`@$ZffyEJ1+`wTStPf-7Fc~Hp~^5t83<7=Y+Xc+mC`qtxH)!sI05R{{*c^Vf2y{GnE5!b^Cv_5eT)Jr`MclL^*oXfdVk zJ;z;t6WopVH(`F+9E{Au$yErUq*2U~Bc9R)DI!@StVq&VheQ!B`D!iF=5cOPExuI| z&|B22i&?DO;L-iIR%cJ^%DlHbxH2g+WRw*QCrNCO>Ks7l#j@-mLDcV$J3IElLL0bj zpCe!q?&n0dM=M?BH@x>@WeezoBEMJIdxY|T0(5V;}yr7wC~eqW{? zo;6M}l%A$LpMzhz-Ra*z#>sL=R$zr!hqf*>KjbRt3OF??!P6G*9?Kn4wR# zeo+OR0E4=0yGO7W9@I#<7v8^8rm6?grAP%INjz6C1Pw_Xcz^c73rEXV^;|#*?Wz{L zG=KoNrw$i1-}@taG2~3g(8eI7Vo^aMFOe>G*#9c0xwT=ysk?Ox5-$k1zj=%f^s53&4nB|`rUIczKsAF zUU4aMFA6%u(YZ~Zb1#f@FM!c8!W$9I^m!XLJmKzEqz%1ytQWHuBrc&En=)x5OnxEjA-6E9F65=N^(eJ^i`hRKMcieXd^p~l%+X5drD~yQ84Zw^KM!1c%XKOIr|bOkVkFze)E#%mBM5xi za}piVODP^I4-=%4BNe$G7ZJbCLRz-M`@5D32lnP>9o&xT?DORbhP0_}+%IBi&ir#f z>Ni1?Q{gtuqS``F{1OR0YsN;!YH?T@7lUuTo z`U0Jl#D?y;?-rT-5&j#C>t|kP&EK7+g=fcQj*mLntbCaPfmtxz(>f>{15n(5`A6J$ z*+@1WP`?onSD)4ezgFZ6E&eMd3U!wQnWE#Ps zF`A>|uDxmbOvIPrw@hJ2F{CHr5J$kJs-HO7MTx4@2{ zo-<1XzbBC!I2r6gcJUdN14Wh*K_eD;QjE|-OeVgZGf<>M`#aTb7;Ck-dZ{=@r6R2e~+LK;t?Nw^N--c3EzzoP*l5y~o zdVZde{1Xmato(fSzdywcti11+-s`FSld-IW(wHqqk_0u}VT0$&2;k>W{|1uYI^`+| zjnG!Vytb~D#^^TS9B2>kRX#yZL%B!zdHb#K0B3aFcrPtxq*5O7UTGh9;U;l+UGQUz z%Y-aVi^!ZdlCLET#{*L>v9{k-j+I@)F@l+uos3wq$5%xy`?99Q{gor*su{hN>|=A8 z-O3N{STIoK1K{C6MB!Kq8SM#^TL7(27SkuW?`P*BV?+*9=zsM4o?0vOfw)p!<52WV zu1^kD*xjrd7|o&@Jj7eHsaGK%TSc$M*6YvMHAx+OfZ{*w7fsTCp8P?Jw819r3O-=- zuOV^7`o9?xkVAMaGW(EZM)y;4ptLzqL4awXeX%Wo68<0if;6$;=pHsV&ARQy0gDsN z;U0KOI9$zue@^M#MFiu8XShmR+97M=;+KKK<&gD{NwRq{VM5M<{ic7fC7S_~eMy_#@t?atr4g#{ZvZ{>91y4K;6fiXX9SIUukTz!11 z{a-!;*!O>Z1eikUg5Iq^D#dM5I8`5Hrx_E~7`VKIT9V7Ixl5{Cp*UKrdbL+ zxhO}-CrTBI;+Z8gdUo3br6Q|w{rZxM&!MuteE~olnV{?+W#)^vzp+0r z74IsU6$&q<@&qL8n8xa0@_m^3jooKw(b5qN4><5y>|f%AA{bsNCOfX2j7{x`fdYA` zRTz{K1rPh_F+H$(!qE2zF zrp}V%QLIvsBb?dNj{NC|I}g6UqX4{@`#)Z4Wp@n(9YMR~3>SF6^v|A#Kk)`--Rf6| zmzHi1?C|u%m+Nd!orRranq7hUz1bB)H_K{`^jwbF8s+`bp5$t4G>+J6>&t`;v-N%{ zSAo@9FL3rrk|f6t8fc&HiBE|9m?>o0=za_8axC#fd+1jOMzX&wxQj}mj0fk+(^72D zo>BwQ>^hPelq3sD90q;Y9c?whu@{P};q}y|z3xlh+!b(+yLe(`32F*%bE-Y0owkb& z9b;n<=Mwf2veHJX9=tx%E_#f^g*cdx8508+)KO~t&8>&}k759OdLg#s$UZ3#)Ih|KuY~>LTZK%ecvq$|6aK_js+>DjjC!(T8)$0 zs~(t7HP)1!5~Oh|6__)4tWI2=ZCm7~Ah1wH@LaWw^o+iDY<1AHN9xs(UbwwFRiHT` zrogmM{b21lrn;&uf#(=9T$Qg&K zS4IfhaQkL7%MRD3Kqi8qP;Iq^Fp3D_r(BkLuvOw?9@zJa3mei)zW9w?>;6189m@sG zts>J1$-!lha)afboRy+el)!?Dkx>f8(C%~0`;~!aQ%T|?e%qX7^nDbxOZlK7cX}C5 z?*m9}whCBN@<(j3FpU&<_ams?cSeXP%seNiiR52Lbht=zX~ec#S3#4d8bUlcjRWi8 z^Fe!Mdf>wdNbsiMX;G742LqWT99DP+0Y36Qu-7lwD@iffHOo5o#@KFp+md5LbW6qb z*tOds9Y@4>U7IWxa_?521g^qslq(qI#Zk%XcOzn)?_5h*jx1lEP!HwaxFW^E=YNd< zo<3K19mk2G{(aqRTitfU@lP;$nfwi~$-Z<(?NI1C+f?MyeZM3nUJdt{To70E%WPmK zPa9Ak&D%1f#SaL`rI zqU1%2!boZ`y3MG6_gUN@&B#Lb0uYgETiB2ixGLjP{mAT; zVHA+wG$B(932bB0w6_VJKqnl!)%DBzwJSPFw5~}%97L*Y9-t`@!l#YLmfEZMaSj_i z`v)Ml1!s_(kSX!NbnSE(7P|H})S1UJRPz{JW2tMiE*8Hpc!RFkvZr{QpIn;1y$Pm> zB%^kzc^5cnUP<(>6zn8A!wuwq(i&N+)cez!fH0VmC>JZ}Ljd8=-wFi~Awo(dV&jHl zBL<}X?bI`SfH6gO%km4dSgWLO0=zZpMLfJUy;y4m4}VAlP<6O)diK2-$jbsz9Vdyl z6;DGoI8)ibjYITbg9-+=Ih&Wf2?Y^Fk8v|0mtkY-fmaY{OykN! zU7Sut7U}-=@o}kF8^wI3m@5iPl}ll9Fhr+xnBHUuW3D*Quks{nfBHX~T z_i>VhZYr`5r;-UCH6cJ__=guegzNx31|buo_o%p*|AYpl#^y8mF%#@w)@%l+JrC<;zc5i@KEMY(uj$qjsbFT{#onNfCn<&+qVleZ!>X zr^A5IC#s=Mis+_|Uc86#DZ1COyJaSJqw~7O z7i0F|(!e!XAV+07SaX{*{Vvlb9D#gBSWY??ciS{(5sRxTY!*vJxf<&(^Ljv*1Iij}Mbo z><{|FC?bEoPaUL={c5?0iUs)CzPw`f;SnxHJ*R7VA+L#FlShl9a^scKcszW7Sw_zM zHAOVts3F5sZ!=*Dr8oYuA?H*IaDn$eOMB_^(Hw3dzEAQtqq*fzs(?|nfcY9bs*vec7PG`R&>z1snQ~UtQaOXvd-^K1cs2^QYJ5H; zVO2O@6%ulMFs-4`%+ilTp##7l-oqFQhU4dy246r>FdmhpNZJ?@Tl+Ibf3Ebqxd=D# zz>Twn251`+Z1ex~{u5IISePasfI3+kLjRvG{e^|;;VyWD(&pJf3i_|_Z?xbkgkx)& zeM8IQrAA!-CoZ%!-jWIVh({||e$aTz|JQ2mlp*i}DgA&XZ%M8vGTDz5McSCBlLakDN${el%rQL9XIPJjw-UlaNy>shjk0Z z$<$!0bsNR`9=}A-8~BKyRp-dl*O6AMZi*eDk%-!a@3Wp)<~I|+*8<4LAI~;LL0-a~b$jv}7)?qsO{$oMa0d-|5K;jZXT1IFfR@xS zkL$6EUW0{|I!oS8=(H|1g^gc<_uWv<55uMH>VlIcybUpujFG*Xmi9E7mmnd^ zeI$`6DD#)QZCMB^h$$Ue99b}?YexMqhub(@`Jns_(yZx(Q8%@?>dK2bp`NrVyXSl2 zMCeRDRc#r1Gr~8kxFeIVMlDZbM-~8gbtT0D9+)FoyKjce!>$W)C(DVO^$e!p)hO>v z^}z*uYx~GK)s#KOeNge5`#S|009gLSh-t$xD_ z&T+uDx}&{aGs|jJk3^WocZ<=HwUxJ51?OYO8A9Ibyh#(EVue-$rh9OocOT!EM8c)Z zGXs)IVcI;A%C2vN5fM4mns`YG<|;5I)qu9SW>Cef0rt^Agee1X7B`g(AG^Ax!<@-z zCK3HmL{Pm!vb9CMlHAIVWz(vRsnR%3)SF3zb-F`6Yv=2>^y3kFeDiW>GG#?@Aw|2Z zt(-iGZRM%sM3N`IFEwA z9GjBekot`+^^b5_rQUVo67`evEu$TnGEfBHIfGk>I_1sXt`Tk=*6}tUANd6(u((OBa z_=r;-4F4VF1H{C+z8L&{d}>vP^!gl~ka035BeP1S5GIRw`c-Rv%=HtU5ZEvq!NBWW z?{oj{R_Y|(T^Tk|=>Xx;Lv#QYwMN<4+ofOmCQw)o%QTP%ZEk=>!&+4b)6H@(p*^2! zLt0gzY*Q4vQ~G3bur?GLep+?{gL$Xa_O&N(fLw)7L$JMiWM3}D_VCKKo~!nJ6yi}vv*+E3}z-a-l9$9`On3X0tO7_Doc+|I-JOZj?h zHv_@*@zZCcr{gpf$5#~vrRIUn4?fA_w9l+#ZSjLE`4zT$|;T<&bO`9KaIuwxmrK$su zo%wrgo3vRZ{IrIMuG*&c-PY3=hNS^PE(j&R(6G{UT%I48r)=+_shD{W-AmBTI8pyG9myPq~@8={zhO7JO5pnuSEF_l|W;_xOI~;pKmVi&)$)nqtxSKgI{=I_w1*%e%i{i1F7VC1< z2;kGK2VCW5Ut)Z9^G*e`DJYrL$Us9Z@1Ows=eOtz;_#xDpAc}))`PP`p|j?|cXleIOZ>gi?bMj;aN zt2rUan=kuyexyGl`6h^DrXG|lw00p?uGK@t#%gul)+j~zK=NR)xj*;lO#On{vOjA4 z_~tfXtm7mVBVZ6I9ZBNAIQ=L^MrJGko{BWK?n9s~0IYxEVoyxJXzEq&F)konK62f* z@;VxrD$sz%4SVmQBN`*U9gK~b{R6Y117Wr&3a39X8)G!9p7KYfoQtCSHD8hFhCQ7e z{F`G%uD%104EFT~ebXa4r8v}{*eGK=B;TXZmfNj~2sz5pTnlHEno;m6EWP%4#aVQj zwg&_t%yxL)dXO96i-`{tZmV4b#bp`DD@$9p9f4g|a(clV3ctF!fsfV?kC-8(mR)kT z{bYSKM*fRtJ3|bsHDESb3R=^w+7=aK?ocpCyV%l0IBEZ3+g7T<91=v)Wov#q?mOi5HHV(!-%ERljb5+GY8`3 zE#exIds%>$L?NL58;%4^qFh3sCzA>@m5AT#qe3+0=Yiw}T!$WOfGy7Xjw|+;_A7B> zc#Oq-^$RYi1j6OX=+TL4Yop7cJt<5h)pMU!=}Y0=s;T1)wdm=`9qDgY1Q(2ZkCyDC zMQ4RcQ3ePCVxa=;der#kUl!hj2M!l)e4J`iQnnkAzG75?PYm`>@V9oJ- zpnH3B<*-qk=yYs!8vd10n{41(bQ*&;MBR)8-HaeGXyr-<28~M?EYR~@LJXpJ6oF%L z76M7+uH6J@gB$I8keA5HouN%vLla!Ftn!9S274W`O(1*I@6KIL*bf%lW|vt&_N0oJ zU$vBowQ<(o&y;Qr4~Q?0?wby4j6bjeoNvq+n0T6c1Lz2U`m3&bxesr)&^Xi^w3|-| zCJ)Fkgr*!oE0!MBvf}l2^D!-{&O_0sdr8dB$m>vai_&u%Lt<74Mw_6+kQqm4b32R< zW1Z12X4I998gbE1Df^4WDxuFzC9n8ksHhx%KwvTQ4r^s+lkSM8mVyrKc{#C~1xJJY z7R(I8p9m&+61Y89IqA-siR^%46Yaqn&~qjD83JdyVHxCWR|SMdsy6Ja$&Xc9T)kJf z;IL|pcHzKwmfLH1kd3iMA`?V5yEC?BPeviU7xdeW_2&`w0Z zq1#6M0NqLsHd>;QU1K4IYG^LAJS8Atn{~r|b?6@6kY<|=Hh;$YGB3c;0?L&_jA7mm zmgbNyHypWUTOKb&%2W%N6ExyT4U;scJRQ-JMH4snq#iWJU;ZG$b4cPJ&^hn5Iu?mi zI#hR9-j)Q`FuUm_QO%(~WK99I2N#*^H-)Qz^H`y&p9!+B73vt=S`$Om_l6D@j z!H>Y;jCuAakb9P55=*8YtbN{>z9Z}e+7-Z581P{~+Z_W0u-}fVJq8TKPpqKuvL5Yy zj*sny70U7p`7_cUh&w|8m2SE==VAKYM!oN}pWyL&kU|V1qJh8%=ahBw7Bc=>=JUGR zH=ZOEJ|W>G6u(NDy+GP?xjnFuv1o;KB>0%BZ%5zks6Opu0`^$ibpFQ|#qDs-B zh~niX{pyFWWxfEZgU zR%1Ap$(>_-Y#Hbx4E_%oJL;$=%U}XGT!72WldH{#>}c)?bS%6cniiWuXnx^f*kFRs zn=!0i%LB@5dX_mVURoL4v`P$`?ZASPN*h>E?hXSB%F941?So6AoZ$puLHVF3D3m~b z3tp}lo9{lZ^RssvG`KpxstB~@GIZJOL}EWe1qs}uF22zmIkal^UGKsvlMRq!n@Tpj zbV#s!NmFfqN1tF@74cJPPjkl@^LPMVKrcd;=xm#Bn>~)H=Qny({B3v#DoOS3R4p1ZPW<3W2^iu2?+@wK={3M~bg&$*3g9N;P zXUXs{FR`y~IKd|w+>77ejR(h2`^ET!u^g@moIA02`w>t6si_GB?#PyFsE_NWZ{RaZ zmTuV0J=fjNzw0YCSh%rA+4Lr0KQjHC+Y}mHR+&c)D@b(Ym98_aQWx}?k%^i5dQf`%{(9J6tZ{K?4=?z`vrhZXu=Gd&n zzbQ8QKNOo%HNBN$pGWQIJHgr53F&ceR_<(|6Kq0a<^YPw*u309W?`hG1YHHs9v3cl z=puDd@WOLh6cZMg^>ppOMMPm{+JU8SrM>DExnMDO=*|rLmFeb)eh0J7i-p%WmrL<=!$e~iPZ~1EMR|GW_Qm8E&VqL z)oo>O)LH)INU7AyO+{*mHqJ0=9@>RhD6g~saw|C00xQ!Kb{uR^GB3%WCe|rRZD(P;fG?&08!ckZ15`8O^t&WpFl^*IOa(#3drOx0|^ z9xX7KrZ5H|YLRejm3)pp?oW!av{wo=VdR7pUd5djwNPB6j`9n{N86E#96VHl?Z&Qs zXsHB7`Zi#<(4PwZjb3%qk57*`WUxcKvPdvPyAFL?i_`md6!V@yn7nmZho# zC;H0yHE$6Fc0GoT?KOl0C*_wxAC^wHFDC(8EunH3R+TdI4 zS3WTBau|_3Os^o&IyCtM$~r~Emb4t81Ue%DXD+^izenf>{HI(Gae`XV?Qn_C9B2s= zJa#&r4mtL4xa z zCW+v7(laXj!u(_0JU(qDfQ2)u_}>s!yr$pYNYYZm!e(CV&DKNxP@yZ@-me;%q;L;97@p zG^2HTRfu9nNr}#!WknFe7h+Lf{`SAdwsThecgo0y+IPeO>f)V^v1}janrUrFiLt|k zYgXF7jY|*9|8=Bt4M@LbQJRzD+8QKyE-@D_P1D6Zr=vqed8b*$v|alY3;%-6nQY@H zf!#Vum~8ty@9^St`RXtm3o+~}`m%GTVSZ?q$e01Gfga!h<`|HJzjMqvtIcFEh!>{H z6r=w^-gZ8du^O#Y{*zwu3WEtH18S=zLTCn^ErOdp+qWx@=5!h7%~Q|_So1jNO_#s0 z-LIZoB72FHFb!$2GYl1P5)}c`6;?ugFQM_@Cke4Je_I29bXk^Mv;|R2P8BQT-GHr0ut@il0fy0O;=4 z)}bC5TSiAG;2k-qIvAd#lV(Cf#*g(LU1rl$lVuekllX&L&7}G#0hBs|FZORK53N{6vi3LlXvz7mTzx{7Hvd(~l2{%U(haXD0~-uF!vCL1zwpK6_z$0f`CdHQJmU zRR&NiF#d8(tPQpy4UsT3(O$+nn;%;oy^p-pfk>QC`h1r+vjkGx7|Izb}p~i|+!(L7> z>acE1OKt!C01Thh&Dgr@IWHmYfY2Ws?UO|0UmK0=Lb5d|Bl8;x0w*%C7?2PsP2pxS z4#>o9CVE83V-urD5cE+xLFA!g@GuoHN5d-uCi#gz_K}Y8Ac8)RBZ--F4+<}Tpo|$c zkAb%kfdkC9anx{eII~iw7q8jL=H^#t0z^=j4N@J}>lugL|1}uME>P#VI*d+8Gavvo z=tLCyzXpNLjA%hK_uC|ST~hs0G^HEpX?n!aIehVc<}>|YjsdfOrF>a&6(B1IwFk#T z&Lf=PugP<(Qdvl@xPLljz`*n;SeK+3QPrL8de>C2UiG(p!*Vh{vV9gFC{$S%n?4K0 zXpINzvJQS(jl7z*?eH4vGZ&#-`VAYQjRtTqw0}65=S+AnRE2XDwN!+R$KSz1eb~8( z+Mnty1vBdQGTX|xPQ7WiH8Qc}Zt9cYu0JxCnot{cdEU~g_cS~YM)?TSasLcG^k}B4 z`ug;?b>sfRg%VVXDf{TNEaXWLQQl#2PmjtAUQa%&J907=)wD$P*j;cD zKxm?R^36xRgyOif_-E^P|94w1IdMMk@>Tch?GN48HWY?k6S|&0!ULqt_&=n~ObYX_ z=6RsLHv~peqkr3eL#;dA=vby)j2FeipPT?OL&aKN&}3rk%PujI-tm>dRHe&}zqV}! zz%)rX0~nPWHg$7tj%;2I2jZo!Q_4-{KaR)EfKaj`qp=8==o#g08&2W8z`%ph7EE2UnEHc^}qx_RW4gJyf)UD|$vKIh? zB z5TlOW0Gb8Yu6#8KM)g-f#m}E^cC7!hEP6@n4)sYth*Ye6|*Ze<=mzTAjnk{kV(E6>71(9v6(cg*F)dGTdl7vbrx8y z4-!PKP-Ha@PrIRneOIM<;^I)Hz=Ux4R&enl`Pd3UX(I1~6R6u=jrjP3wuZWzf*A|- zFMkUHs9TDpn8QFb0=c)+!nL~$ljIqh;i-`8#gOdziWZlKR5pIMmx|U@)p)nbp!~~8 zf-`j>h*0yAFx)3vxo^T|d$6iP*g%B*lK1)G?Cjxs=FpJVf3oytW-TQXUU8t zBzIKDhbjrwd^g#Ms#QhvE%E4qv)i;2eVI1DbADIwFHbvGn_;)0_7_P(Nr0H|#j(+f zv(x4%5<7gvtco+D9;zXXREFH2p> z?`JZ!Ay}%-iframYdCKT-h=0;{6*Q_Tvrd(Bx;@uB#i*IR%^XP;?iHc{r+PvN_zZ^ z5)7bcg!_&Q&J;4O2!j|hZGs_^$o}PR`IG}&k*KeudJo@_%`*`=E(Ug z@2}Uy3+J@fG;$dSoNV&z7@fHoOLi9;TbKIqcI?BA;+wSlqTIHpX;tr|4#tAY11p*d zr!PdS3i=0#2I3A5`~{BvvMh$4_rjzhN~~v1?GitiIU;iQE9}ksZgLNjG={ceJW&qA zab&&!QS&v-3XP`t4~G*U934LDOOuH=_x|?t2pP_yK`pU$!JL5s zf}CnW)b1|E&*l>ReJG_v>FSNt4kPYK%P%cY_lW7bAnfw#hxb3&%+mQYq1=Q}Ww*21 z%bgDZ5{|K$2-nq{Gkil6{ld4J_bAK!lSYzWfz?VVJG8^1At6d_uiWgj16wl7$BGZ`QAX%Gk3%O4R&Vs4M6&{(gi#54wIj1Y8Lmtb3px|Qs#>c!0Esh{b<2JE8 zVFuy7^l$M&G;^YaU$6*n8J@)%iC=yj$my!bw7ee2e>uV8cb>vzWN&{~IRmn$s`*rPtf4C`dT7HE=jk=R3*XivNa!jCFgg+^i>Z_z-}% zHT;hZ(Yj}6IGwajT=Eif1`HL7{|prev<$s_v^3t$m(fg!KMEuO;74CVQvq#F@GvfV z--6q;$4E6C!o)oPH9{LUhP47jKHh74?G>J=NC-d{+xZ9nbi?O@o5S#M-d?!ddEoOs zT@+srq+Io%tKiS=AN@sMs`+2j2r*}dgXfT(makcu=q2g9=J~NAkK4V>F!&w6f49Kc z)#BCn{v~ySm22DXHl(#8sPC`TagdGjNx>*2A6+p*(pf;&F`*xAwOaNB9qqF-oo`ic zl?Et03e~w+44%5DX1kUYdDIBD&%Td z1y&tUa5GaiJo%M9nZssl{KbD2ykilEUVEh_YX`yNgX1#CNug(=NxEnk3sBwuJpXEKDz z9|$eRaK;gA;AKh;NK>H|UcQWjRNs*Ba(C;BL%z1fVg-E50S}8sz3BtgimQ7G0aypf z^lJ8kfE?G9vrYAH{SU(GjzX4q?os6_j_fKwb*|HdD4wXBSz2R=Nh$t30S=z+4&dN% zP&GSnQ90rhZ;~`${%ntF=H{eTWo^~UZY>YV4E@)SEAB1ZZ1$ao!s@&^i$olHL_%}= zKJRjFb!2;z=x)c@hzqN_uO>HC7rVH-2Qy+t8RfuSYI%8^^R_ef=;@9_*+0R*TvEgw z6X>{yBu|vDZTCIenViP%=f+qI*Sf{j4UdvSW@dzfzdC)FNzt|0KKswMZ2Iz2xv?Xn zY1H$Rgs=1eGrUgV?XFl*YFD;iBC-bl-cpv#R{x0la^B7#+GYSz-_axL3cpuVJ~+i+ z%S8k7;;}>J4pTdD)%YTvE1QAB=3z$FyJ1BR$%KLB17kc#1bE-(~N2nE24+3Cn= zGQ?BFlZ4QU_G#)3M9mJs!if6j&H2|C*81ZMcbG+L9^4cRYWdjubm-eVi>t`VZF;!7kq8W)*bo(ABX zMYf7gU5Y2LRGt^)Ez4810uFFuhI9PCcN8|Knah>Of}p^p21^y`H!|qL)v^Ge91J$= zscn&(8XlPhmqd+?^y`AV>9s~T=;bsVWL8uMA%vYh!RZCKpwjU;xLfE zJ{3fc5(7tz0FzXeYB&3Z(!k+-(Byr(zTsC^eBr2BiPF9+@!8kU=|>NyAX@ycoBPR= zEEhjqC7}hi7W=P)IL1H5sSPnsX&62@`kU!kE7E%?BMzBr_BUo0m?Qtxy1y}7GB@SC zf?oW+8HYEAHfG0~uNGOv@wpcF>&qzZ{?hpb(4FxUd`@cZLH? z21eOSP=J^&3-}XqdAF}`|9}LLMU!K1U+1?^BOD8TnRv!}a!zh9NBSHIn~GXdqlbGqKPqtx|%UkA_iUN4UYyFG8)Pv&W{a%VzIH#()y zjxTq!VP&b36f`FMbUVLJYD{dcSB#ze_&yUqIWfN1ylf|FGmhESaq3!jBGY4ez$-Jh zjf5y*{V4!rofZm9D(7ZV;aFwsRTtTf& zkwen7ALB8i>RBkQit&jvtxz@2gTJgxWJ+9GenDSOWB$jU@8sedv+vWHoM5zDa>uc6 zWebPklGe+Jx^SJ#JdGNnLJ>B%&UkucX)jfdNgAUG{jjeUv|c^fm;%T7a(jTnU7rL` z^D(|AJA7@L+Mf)Y%12$JU5jaHq6=jDSx|nOqpf~cv{k8Ten~#-Q5{pyK#I+KuPY$M zW?1o1{-m!J7AKHmb1mG}THmYAulh&%s{*n@K-aIUD17D;JH$}iimi*-q$tJzev&#;7iOItmjYuE27@QgsDT?HEaxz(Vk zMlaU@f#J@b``%C?%&xKTPBG`>w%Q0Ui{|{xD$#)`NkP9UuQYPgNcRknB6E<}FayJS zQI?wghBxwL%L@af>9N24RiH zbt-n{W#7hd()q4ggCC4nL#-kjZ4&Md(<^GV_$3`LsAzk*Ea4#fl8`bNo^H(QWe0lt z=^Z@#sC21U&rX;n!k?|VAiSFeku~q|?gtpM>tw!9vQV@J@(!N1pQhp2;_~7%+66-u zXJc(QGZh?!oG*O?CHmq5Pfp{bePK}}aWr2!PgJV>K3W@6Q~9Ijr45o|-=7BxjP&qrWT%UzzhaxU9m3j^(Y6!9_$Vcp;_8vNHIj zYYnB8+i`Dl@VqNCC8-M`-&@d-Q;j<^8bM6jhvz)g_}-(7g(cU zf1DR3*SakqVkrU=B5z=+rM#;O;&tZwV1m<_ zul5XYu>ZUPv3FLbI=33KR9H#FztX0{Z|eif}f`AQ7}3Y6j6yiLtfG%DH^Y0yDgqFISZxs z+VX0@?MWE&sK@zp={!olX=uTN>ATFUh1byTbNcZLA4^8F&RmOgSJ&>$IzdC1^TL`6 zpA@q?Erqto(-Xad-&oTx z_La-9aqNvnDaAASKi6jd&zzX6EG?F^-4L7-pqXyOz@uUmd-yuPo>=zYY6sL$vHsGc zBW!IaahGibfwz(c*B5Kp`V-ETYOuYh$6Wgq2v2<=k_$mX;SthXXNy85l9{d#G|V+P!A6Ugn}YinHTG1qf2icG$OmoZUH$-Z-2N6VkHYj96758&0dW`ycBV#8{vSI@CS~wp z9Q15Bov~U2xVBLk)5s4@{R;=9yl{m;-!#2DOYZ8e{Q3C|fuNswnFJ@L_H$|7geQzD zi}ScXOj;cWdl)sAgAx(faf3n2`2&w^lnefEO_3MFu|KN|BcrdUS-T-iG}$$<=|rca zY>%`8Vq+W)>q~HpDc5v-GJ_aXpINef)-gy$7p9QnZ_!r}AHi4u;<6Z|Y%l{%;3E(dBMVgSml8QR;r_ zBfWB8nkR%pQg>Z^pFgw5;s+?uL6f$p{8ZVlLQUE-XV(9J2}2fO?Q?6VLv8NX>0o!DC16qV&L zYid5UOtgkqup;FRvxFL2L74%by9X$`8pf&N?iS^~Q_CwXNw`PjVamQk`rI(}&GO<^s)&WKXWQ54%?U@ZIy1fmKWO;nsH}!XdiN=Dq4? zn$wM5pygW{>aw1C`pY|2ZKKdwqxkiltrS&yriGm)lPzZq-hT#il&o@OaUp#F;o5EjhS|a>37%lBBouf&>dDja4GEiG%QK0_v^;6Uv-A%$vvftcdwiY zKOO6-?rVajLec9cO2vN5b>|z4HEqnMvW9<}OtJYa4^OF?>_wd3eF?uh=pXH*%=s?n z5_nx9;CZP3|Eekl4M7EALw@QG0Xtop?}po-)fI42<9CA>#X;*OBa~MMB{H;BhT|MR zo`dXJ#GZ!MrL_N@A5daSVspU5^#RSyV~9M;f1Jy2i0?eUj3lw#z*LUIBl;h^lFAl* z^dPo95q=$-fWB|)jOFove9eY^6nyy(_A^1_OhQ7)X&5>OII4Qg5!aLf=2vf!{Yx}L zWt~}2#gAcxG>o(a`r=<-908s0CRsKahGVS(84pHNajn!~W8phO(lC)&^;x{hxoss? zy4$;Qy&34j%kwZoRuLyk)!P9gL}vgFLEtN5aDx;PGGvaZUhvnRxv$Z1PLwgVr@T}I zPKjt!9t74hez-PPn2;7(+VkA3EqDHND&}z}_WyB_=JxD^3phO`R3w@>$g#P>fUfa9v@149Rv%PodLVtZe{1ez`x`DV?xn>a`5R|-l~$BB zk{HsD-Wnunbk{D+zJJ^|w+SPZicdN~_UPKQF;9d)`fdDAab$*!U)jFc(U~+R&A*wb zweNi#eW5_O7q5L8b*#EwdtdNhtKu`eS4K+K=B>%|t)qE9$Gnwz?}b%zU!0yg5LoZ6@s09w04pi;D9UjB|^+NnHGIb+`D`dP*Sl z;s|hYOwNH1{{OB*%+xS@O-`T=bu?^w$?d*;^T9BvhsH?RJV^kv;rTbv?~wI<&yRdIknP9s4Hs%{Z{eJO4gyz!b|>``;3w zI5YMHfL5+8NVZ1#Fg3%<-(i>|LA>fj*Q^BX z|3|YDwErK?8e#ZfeH1JJD96!OZV=jN|kjRlwo+Zt4rKSRcuhrMq${#_3WiBvF<2$8XQr zI*eM=_{Sb-$E)yp!2l5%dqs+OBwj&GYq$No%EN)+LTrBnmVPGOERz40emIcY|J$sA z^hen*L=5I)YQpD<`ky%gVq|%F1tx@nzU*dg*?s@>@o0$e#rnACgP=vf?6u^{uR&7! z@lCNZ>s~8LyRu&EFUMavjY376SC6yT8=jKC#EZH$23b4Nlv1Tt8PZcDbJdF|3fMIpXfB-4R05x>{1^_+?7U) zUNV1lATy!Wi$v1q98M80-*iR-oFD-#q#qzfJ|J)`Mo^Fbw6*H3%(y7 zGj+M&?k(!SoXr!z-`tj0`+n=hGYuuOXfQl-t4@6OSOBr#ZrlEGn-xYmlK2MO zN9_6{qod2}e_;v|0H)CV15?!5G_tOdP?w8o`Z|Dz-kKLM{;n~w9wo6xsjlvNzx{FC zDY!y$Gg*Wj-VE-r%m7og#YpKMY?e}aZ>#V@J2pw6Kb0ypBB3OvGv&X?VoT;6I zF45A_SqemA)8912G9Y`-^Y&W8sewC+#Oycxs%^i*!Y>=&LiG#Uq+Q#8dF23FHvhAf zUw(6{xKq(Syr$>MYx3cv*QfT_k67ay4ia(asEEALC9rNZO7&;j8AB zcRzdM4pQWF$+h1Q*Xpu3UKVQL#3DEHw3=3e`hVFXLs2jHH^iGN^K)<^gJQFEtu*u6 zc0fb2wcNYJWRFR&FV(Y&Z_X*Q4OT##A`Hn_-xE^vyDJ0`S(zcwT*}=E`k5!iRa3uH zzYiY?>dC#ay$^T4G8HT$?ZrDoH=Z{bTR|0N>`Pi>Z(chgD$G2#Z^TY+eBtHsJA83z zEC2}Fm`Qyezs~++9V^QKKorPWEE+111JAJl>2_}FJ!RFlRI&mp zST(547tG5zc3WF`y(TI@VMA^Lt#~T?YUdQzZ;AKU7qC%Buu0Z%Q$@(1pmwOMp3ASM znIpGQkXM@-2q0h(IXv^=Z;w7OJ@F$V?N}`MFU6E4Lb6@to9y}3Mn-_mDbxkim&zK+ z3M;WM`#-(Wjef7|O?0G0t3kBApU?I$ZQ;Oy1-}Qdc!iq@o>9+)iFJg2y|D&*DrH2TH-e*G5==pi0ytV zJ(E|-{*0IS+1Sv{!`{TahV{BWl%ZNyL|9HI(+w5q&-Es8O-Zkg!H_gb_Cvu{k46>* z98rr|RDX3z3VlaJ*oXy{wh(COcTaK7U(<>;7Ik4C=O*L^Z(pb_1wiPD^}ZE?RqB3* zx+OrOqsd%sQZV%%_*IS9@cVN|w(Atj?BuFMi+I~;1sBW?kMm_qU8RpVVh9FhA3Poz zn`psr4P<_~YN4*Z(}h(ak-_j8{QcP zTAB@x(<)tTCz_1Ns9*PNq?~?lcSLZ=E2c~F8S%$yzRo0}-u9NM+8Tghw7c`16|Ft% z%d>yYA;c|xe@h%q&?ydc)A&I@T-}*> z##KM|d$j94;^MkjPlJ3NL?vew`j$hO8J3g18STE<326F~<+yPAAW@O#vc?YbyiG?C zdlZjO{U$g^6t78rOAtqt-fBWhV-j48`FcAFUWz?T!VOO4ID}7Pk*)ncb^aGv|Ar`3({yKlZjxSBttCy9J}WNv#( zLnK2+&)kRZ#QEFXCw!_yn_uC(HVM}9F;33D|B!1#m%=Ariy-K`Zb6Uu{Bu^N!QvLI zGljbI0M?Bc!rp9CD+E=M?DA8q4x`DL(itX?`C+CUK59Z?Kw9j!AWo~o)_wk$`_Qjf^a|W=R{q!_{{Ak{N2U4LLNX+6;&2 zNUtc&PnyeRlNzm5PZqX4#bk*BA;26EwyYNPz$Uxv$QZ(8yC%OwYunFY$=aW!pVmEX zbdxCZk3rk2Mrwyg{iJwBxk$>vSw^ye)hBIC=!~uAhi!>kTI3~@aKEG=`lXAoHQ7qtvC?Xeg4wI)D!?JZ1 zy~D94d9;2Y701;psAg;P<`^moT^xE4l8R#<4cI?Yj78f0%`A_!vrQr|vsL%UX^yj+ zy_0}GeG#}~TX`gx;hE!l2%N~ok6cbc`k|_<+7-AU>5Z8qa;c~?nP3RpoU*Kg|GPR9 zzYo?Pd64By6;*ynw`JsDl;kc0U)c}32^u2D=w(8YInbj!v4Ot)LxVB*d71~)DxoRUJ>Bo|(AVy^nWdg{r$dpc z#scqlaaD{&L~QZ-d}2Z)2p_XZCaUA*`iNMCb+9aKA(_2e(lE`8#4<$!isom67N>u! zgwtZ)jvakMj;$$Do*pv}!Per0TZ{^H$useRu_MarEZGgjpDb%utPQiyymib}ZiE^% zeJl{2mQ(UyP^i$${ZdFR*ri8uE{K?frwGrOy7k$+SOZi&p<~;H6+)3#-`{2&{fge| z)>{iP(sQ{|)+YcL!+lv0vQ?y3JyjK@Prq{f;mvBUWejUiS;6`SJGn7Y919Iqx;5hP z1y~K;$$LU$&rwq{TBk@u33yjW&A#SJLG_et~T%AQqMg=8(mGcLrlI z)`^eC zAA8Aes;e^_*#0`M>AbDDJBsI71(jW9o>@VC*&^}N@&{?IWd@UHMlBxnS+sa%XK0?% zDqlxJUr(p<%1|~&McM9J^0tg#FGANmWO-`$Zz;(-7`4Ufd66=D%Hq|mN)nSz1{EN5 z-EA$nKP!aDMR%u|^+dbCSgIg!{v=eLpuUQfw`c5|6H9_=;g#Ut4@T2AP&)$k*?t)~ z_wP#dnjPO_E<7>7D)7|aK9q$ae@=Jn0DH|vbg%!lEm47YspVH}vQrqple($F-M{lan9LDdg7AxIT+&T-}}3sEeriQ4xrge^97#DsjnGDVHCE zCk$@3s76lSE8992=VZj%{PTe<0N9sXZJg*9R+!4y;C?;ro8}VJoMCT8+ zpNYtsXHKH8LP=}CbKP3t=LVD-ooRK{7m-YDCyTUWSWb=nAuQA=+x>R87wQnYz)_F5 zKM2(xA!442Kk}7Y++&qka1Pyn5^Im+q@nAO0-k5!F;9!~)&j7ho;Q?cY~dAW&V<$r z^VTHrFbw0wP@np_V=9mI6A;4GKQ?DX7OMpm`_|X!1*k@}p*V0f`egLZfEhI|U|eAWZ%w5kBhr=EpFk{JI6nUXt%BGQBO!kTe3#}5Au?KwH$Fd{Kx~)B zGonKwHqYh9{ua$Y=iq-Q1O7O*;*2fb@~Jd25;njKJx#%5=-GFnTFk;#({jM50?PZ=o1OJCMK_ zMTo2#ITiZ5mnq?%md|YM$|HC<({RSN$jo5f;P`j$NCsR$&WD=_xPbOh@sOTUl!p<} z02LU%e38u{p^%=_hkW>e=Fq<5NVYFY3Zte0D@1KpTM>a6QhO4J<$IGEmr70PD=@af zF>gQ}b^Bof!)4zW^wgB?9aJ5kcaAsXXQ>^{Te*q!<0`e;+LBc_wCynpQW_Z2&;2r( ztx{p|wk4iis>sM$kgCYTT)ibg&MPP}OrcWG2NuuoM_VfVd>6ls-g(Q2Y(dM2h(_8~ z_+KQ0$kqbN!rBR=RQOk6RYady6mKtvg?JU_`;9M^ZtV-qL|l+cC%+?>Od3L~?8hGw zLs>sFvJW9YJlo55o1HA{o23bZFYbib7Pu%wpDvWw*hf{`fpBCJrn z$1ZD+KAEwz&uv44F2y`&#o?mc3l?msN{vWRzIQ8WpxerYO*iBi)#f;d+{Uar_Ng`& zI~p1Mi(x)vNi7(JVWrm>`}aY%F>%3V0b}qp`*vJBld#o@)E*uRD`#fHg$+y;AbEGvq6^M+79Pgo3+PZ=2_WFw;D8+ zlTLY)Rqebau29#63J=YH1cb?`7`u^m;iB~^H|>lG{*HOuLc0}p(^h)S#>AiRHBy4p?1 z{vxRK;%jqxd+PbJ+3GH0|0%rGv!a3s-qp_s-fe_!0@ovm_a~N4$LS5#2{(^Oj!4Rd zC(mABMp%N8me$v9XXq^rVb=1XyD(JeoObT;ZrBNgMdeA`RmLQxGgu(i!mE$D>jDRK zzCf%vx^L16v4HD^;r+M|&RFe72G`>5%2bYyhiYl8=0))`rEFHY#&a_|_=IP6E1|g?E3pUf;b)NDmxZC=pdB0 z)BgMXE&Mco+t&AFeIftpmVwy(iQ5zp{MSE6rq6k&D8W*Ehrc1>!9gN1{W8uB=Reia zBR3j2J#-~IGdu1rJhKM9_2wD#=@EgmEqG@qoZUujvWvBuHq^7{PwUWN^N$k_g8G&J z0tBAUvVfDPUyU4VY^1*|rOKbXoCMsty zgHYPVx7M-vTD_tA?!uoBXc7`^f6za1b%oHegxV>#v3y7$~2w5;WcOb9eeD*7BEjB%Yr{^zZSjQW{4yCGveL0h9ZqViC5{lazShISVp4+EpUdo+6kPL{fdI zV~b_;Z*QVCK~y{Kv5h3L)rV}cw{>E?aQdQWKJ=1IY{$%=WZn+>;EZxjY(ubj$*u!h zF;v__pFC8BGgne^zEFX|l@K9aLDz8pTw^d-()3)>oXeOw%Ml=fM)*<$5!w)*al{ux zhzuLV?|6#HEf-<0Ow&LSEz+Z&2webEVc(3{Vk{u`NVsu_v*ZiyEs9aG`TZl(4Bw`w zj;~8kP&F>~$KC*AELRjrdXIVZB73w7E$qC_(G7%vQuvp7opEKB{yBv4_qPo}-6QMZ zcHXn8m-i3*p*uNnXMU#;6MN9ZQaW;jP7$4Ximz;NaD1jDs||^Rr(AXt(q^edkK7w@ z`x|c@%k_|RW#eras_E9?ebKjX_jP)ai+d&buOICko~0<1;W>$OQ}QoR)U)*AT9HaZ zDhq#;=j%Ai3jHoYj%d1;T(2sdftuqwHy8fe*gy^A#rF|IKj$(@bye>R*5HKqSuXvY zo+~URKYZxgm=gASH1$zJo=x!lu4h3Q{b1&NH9ni)4;RvJ{>l6}m5wn_ZTDqyVeY#- ztxV{a6m+@RaktI21!q>x-Pn~hpH>TVx<&I+#+bX+Z#9<}O>(jle46f5bX_3aKGRP5 zIh`IBm74ljrvKcJpg@Y}_)B zulwy{=LnBt?8`6eIp1@mzU}Zs2UNZ;+IMq5ro6yMwA0a zl(Ae4MAmcR6-%iVsU;N$EfohN6%QdqUn0nN>W9LorE_OXZeyQl_&?L%BuftD86-jS zpFyY_cp*L%fNQRP{Fp;1NG^1cVK=9J_#v}H5qG@_#dj4do2WNTt^y3j;X&dDtoLLr zL$LX|JKuCFHM-$=`2`cw z%A4%D16e=Jt9bfpnmm;M%>|x-U+k)1 zgGb1dFu3E_y`t2W2W|A_v~UiJnZ*%2NGWqlIl)i^R+R?TR6zOJXlClFX*%_ zoXJztwZg74KgOGPc1?=EwtOyCTZ=~9V{>(X51bJHU=53$HNk_`B~yJwy{BQgh@s)9 z7H~_=)Vp|DTR*5F_T=14_wcsQ_SKZlY%?~-O>IFMk4B5Kk;zA)f(KzLL$$H03-hGt zihdh(Wro@Tukq6z-2Qy3c_^UtANmmzH`VpA>_q#rvwk8}c(t|qgD z+*HjnGN${9!HIN~M9Fn5I(6sfw|qWkzUO8=AaiZJGEi6hprPoxG?zvk6jLcKz zrWgUW%WcDkd-J#pB11No1PuxI5BspiBJsi1oaU5ZOumZ=yASSaqxfpCHa`;gizcKe zf$E|L++-J1$B^lXWXtApOW&aakHxWw-j7zU9q zF0jhJQ*X!WYWuhs8jPI>bye)1Jiq%)2qM9NFMh;2d5giwb2CDi4!qW8j>PW$xJITd zld-TzYS}Jw@~lV~)w{R2?Gvm*-wMJOCf7~h_cbjdfIo>7(`%{{x~yR0OVPO4?6HprJ< z3Th4BVtGBi<1Cg8KgqR-Mx?uhp*Io|%Fku1Tp_9SHsm1rermQk!uxe)caU=0u+==2 z#znIdzujDta(cIE4-jU(xb_lS9I89)fuen87rq}L*>I*!LZ`PJy_+0OK*_~vEJ|?c z1VM<5v5TXnpMC#i__72Xuy;^zH2(qGKq;{2||z7lU)wNKT~Ph5X>?e=qz zoZNVIb$+v5nR%4iuP&#|>SnZYew%$fCd_91z3C`JE^bVB>d0eqaNT>4RZIC zD=x$4S0l1;y=Rn=s^mx)q>}3_&Vy2Pt}uGjvLsYRDueZSd2K!q_M_(;u4eJ2kEHb z6a`1ostO#FrTQ;k>9im3)urUWBBp$`j`#m!-(HbN*0{Cqk5nbk}M6QnuS}pA{gBzpSdaQA$ztgSfp{O0FMv1!b*v+PwzF3mmiB(f>%P@Gp#FgT+ zdXwKrn+P;Z?0J{{qW7MG6I?s3)0EpCUDn0hWB?N+7dY!b7+YYAM88}(W^d}MYdfpHh`Sl5hm?O;{Yv~z7gyA8ruDVvb!p|5ZEG^LHU6#8zI^^WTx#%M-lCi?aQ@c16!1FYvSp`-YEov9`*K%AR0^;JNx%q4LJ3=L}Z(33Ry(#;Uj-91%yG>21RjFRtx zZS|U+JLPh&&L_LPGWMSDJG}-Fj@AqxQ{8Q*^-WFO3i`Q=H+YW^Gw=>V9N{#b$Zs8WhhCrYiD6VBvN79LoE3~h zx&}L&@A2*;$}&to%GYV9TYrP-Ee-0}g><2x@DQ|xl-nc{_CHvazP}(+NVbpD`)D2Xygx*jofgnTy>Lg8Om-E9sQi z*zm?^^ICzkbJs=Tn;w{UpQt#K$S7_`7g@m#H@x#j+&)?4ea!cokbOG2R^#*?Kjw~X z>Zm`4_GWPI@%E0*3<=2hS!)V$(w{1@rDZhVnh9BddtURV=J#6}b!^aCdq)_ES}nQH zfTYtIMClUOmB#f#Mx~r+MJYQ#5Q)%nZxY$o=rfXD=_iuR5GzCDK`2T=*b2|UJJ*G& z9_e|dROKNI3m{+n@C3qTyAX3yyhaP)%KN(^(Ud9jrG>QvAk=R-DW)hWe=Z2FgXaB2 zl_`B7?eg2k619!ik0@9kkMkqA)9$4Q%@emXM_`E)4LWNkSUZp8d(@G;^g00hfK)FV zi%YsOeIN3bofyWGZg&Ks!QL@UzHbEm`xkBj5DR#WQo=c*_voEOGfIymf0jX|88`@AjS242aotPbZAlwcVh6qbL0M<1Z6)@uRMkU*t4g8|tTzSuk=@ zp_40~OiieTetX>FO2JOV6~UN|PbjAqqnMq@s-)H6m=)R1p|(I1wwZ;mBmkY2z%P<2 z#zjKJO5Rfs4@b~P!PDS?MyxrHQ$2UHu8U=HPSlw!&UN zFr}5&f}QT0ekMwLNxYHa{X0e)!H0;RMy7eA;5O<*i+n!}93$vW9NYuw`*Hg~|E1lX zri|)Bt)tNZd_$Wr;G$}GzazSO(IHYFJ*9q_a{rWq#C>hC$Z>IX552iSzH< zLX=i&1Us?v7OpS+-F||JK^FxUh1u*cS6_JaN4w;!N>pTi2251s%4d=>nWiuc>Jcis z2unN$-6>QQ<_6e_L@{=vfhp)*v^0~~-Sh2YG__`mu>^A16MwdahzX+Q6dpNt1L|Q% zE#`9v^{W3nnFHufz!@JNqxc67a9raa7)DV1uXC)502U6hMkkUPDHI=WhFD>lP(At# zVYZdf8nB3iL`{+gvWP~>GZb;S6~$#ieWlA31cbi0)^a$S5}$|MMWMby=!`JpWvfHI zCYtoJ)X?NK1wqLBGiXJf|4C zDl@F>O6`w|jwDQ`UpkhKLjMlH9%l|Q8!$AtbIL;s;6@xcwuSGeT+0QfvvdmA+N%!z zT+evngYgSXYFKtD4}P6gb+61hG7L;Dvso7Nm|JP{5aYyWVf+Itm4MfQY3lmNe^o(1 zq)kt9Ta(b!D52@L8G-L?3FQy9qC5H*zDTLZ_ETddr@1|uO`{fXzltIyEs%Q}WI^C) z6(^2o_gr7Uca1yq5mdJ~`x+{r3PcVGUxw&4?EIezmhxD!Ga`}_zM6DxoUGDyiK)k6 z4QS)CL!jD`rj$@%+L6U@kl=sQNWdc*$m6niAdw@8{Lr|VzRUPmGQ;U4!oblx4sl+S zxYnv8tY*bD&tzM@lCuXX*A*&PlX`)P~`|%;6$6$WgK0^;riGgCA)0MRkEZVt@wYNgi>=!R>70=ppT$` zaf@)$7QeKe0!0Btm!9m%@IIBjxM%C`?#7Waw(V4T!DD*2Aw42xYi3o5IBDm;Gk;gZ z$iHV(Rw{bKoDoGN^~9k-p3xG<{cj;z^05FCXmbjSYsj&Y?Di_S%|sMCo{R=vXI{<2 z{Nqur_r{0HZ%$|q`lA!cnmvC0-;=mR?6hW#WE9h`OEu@lM4<%1SG%-dfKl0NDcsv) z*LcvYfy?$ThSq$Eyn>9h!zuljIi zHtM0dDvn24d&WhZ5NftBBjmo=R z4l{Gvf>J+tj770*NfztUj6A3-K9LCYe-GC}Xvjuq6<811yyqk-pbY0Oy)VUHf1z1z z=9!IpQxH@3%$;Y92qSK{yzsxc@yru{(y9yD?%KI#g!0Us5KrR>bWQ!Go#MEtjNz=3 zjK*Vaaa1Y3>7+8j=PN41s%!ZmDFQp-sHwlalX+X{{#JLE0q1z#uP%^?oga+Z-Ih$h zCh${{dr{2;iMLloO)Y~Z6OlWgc`x{qdddT3n#(6>AO<;LAVz7HEflvCm=JKpHTsTW z-USB7M&zG?Va^YOVQ%~RQhu=W3`Y&eCb-K@uE#+s1uY!n-NAb&1mER}vpI|;&<$Pv z8=(g5uIZaS;qgTqDC-EsE|PW^py~%?SLjlZP;Cs28COX{^*O2PTP}6Kl)UY;8zq=v z#}wK{zg!~Tz}l@8wwa^BdA()0E9=Fvw_1aF<90;5i^aH|hm^d<&axm}l%!Qp2r}$O z7`CBqFQ5+ow~x7WfW41xfw z<6H*dYQqGbJlGQVpjm2gu^c+!=7>QX>W&+R(4S+&48~SsDb!`aKGoq0t}|2b2&7-S z1QRRrKjX|$#a#{~w;rLwhV_nV1HR^_nt8A`((Lx|0ZQFvp+eV7hfpzcH2;yj2x_)8 ztN};(K*=3EG;*9>u`iskmj0iwr8pk7m-X8uPaD%FVBC#ESJApKwGls1B(9}5o0p7T znZ?r-c+?Nj9ic0qYx7;}@U0Oeo0b&#*3(&KFKL=ua3600ZQn%`-Ic^Wv*>47u2%&p z&N*bc}{}O$Ujwuzx$ZQhlvJ~`l3J=BlocT zGF#)YxO1g@BBL``F3{niOrq$%sNXwz1@nFF$mQkGjWU%>u)aoPF!&)MmsVdaTMg{c zP;#-@AiNa!9lQc7zl>Cj+pfQX%p$B!(g}aBD6b%g=F#+l8x{+kVMZDBXkIv6t*hg{ z`HF<>o0UDMh4m@P6YXD;4mdlxp6O_C%5$TP$ck{!)7~{0g;=(Jc+>g{gEnV0hs-u?$1uL6_E)_VP{Jw1BBqJTM_vitea3K^i)+~gFGQsrE4tKMn9W)JBu*WVPU z)@~W*>?v7yOe8EX~f<^!hfRp1(?p^>yO`SC+)rjold{MWdjIOcu z@X+cf3harwgQoe(XAlVNnGB)@4F6Q-*lqq$9H)vfRyy-L8+l5`__LaADe+N4TE;(P zg)Kf9ugCr<6uvO}hxDZ^oD$*)G2WeJY*L-7p$fLRGGw|iq%y29A^Jzkm~b6 zwCd&wx@^*K(MB#fS(LEG(+;GWZ0zp6aQ#vBfN12B)5tCm!m8up9Mj(yR&E33xdSf0 zo)LlLbYjMB`#?i+ffdhaB-pWM2OY!Q2L!lA<hNUhgj#O zhz)0OB}k5A%tai2jEOwSMMA93M1^*r?$?`ugze;zru2;r(UyjAX%+uMQ!*B(^R-l% zr(@R1VQ3}^>o&(3Vtw&J&-&3q!nyIbqDzJGA3uYNZl`l7TMH;yazYJRXsGc3&uD@a zuJ{JW-R(TM%~dZi4?Rd-9YUGX8;>173UiL}a>#T^I64{1d*O#cGP}!a97i0pd#ati}}(XdWBXCYeZT; zdM7CV-%Wq5%|E8!i0!|b{;-t4re9*J`nTYJ7=8;b0r6!J!Xmh1nD4@Q3Wq^J!$1Gu z4gbYDcro&Bz+HcO39=+2`C-s8E0Z^vd8*e8xG^r!`>5Dfa&Ux}TUI;zA(vc`RXt#MDn(cXSEjzC8Hng#(HZ^6hEZBg>1eOo|(n`_!NL0SFLtE4zgb)D_h6* zL((pYOkemwTHgMEu!O{Z{&>aCJq<2AHA+6R1UfG<|8iam!cwFBIqF(k_bEF9y2NkL z2LH7U6gD8s6#YiRl-M;SNW#QoO5zzwL^AsX0`zwZX1qYfoSi1&YdF4pmN3p33G3(b z?UhZsFzMMp@}AzTgPz{-LzX3P7o+K6(*6RSeO!?2s3BNNFj$g30_A`0k@7Il=#-`{9b*;!Nj zSo6bw0Ja4Ei2PGClIElkP#E8GVvs!c@q>wCkZgfNj)faiL5_I`g6%)d9;Jn+)8$hO3-0pv8;UzI ze1L7%aLG`={<1*Z%*U*r1I|#@*FpwHPSxevK~rZms?~W{M!mu>WGl<;X#~UiGc(}1 z0p`g4l*rUlBh-Zl^}<{ok9$EsCjQr#jMkXg)-MW*FNz^~5=(sz@=ev-g5>2$M*71m zny6`uY*w5C`Oo3ISwAY%K(Jj?vE?Y*j z13atj>uuQ#if{pTHkCDoq2G}y=}?9h@%?O;o+{~2yE84N{FVt^1!kJ?3!jA~RD55x z=4ZUocJgj@kuLD#kXgn&@z7;dK~HGGaAn)9b4#4Hl2Cc58zwo(F2xha`2-3mexI|y zmU)c8lYjka4so9r^S@@ao*C@BT`JJEM2u7(8?im{xyVHku(=GZU+M{ z%XSe{#J*?Hwtfc-OoIS$l>#&8V)TWrMu;+`bO-^smw|IJz~TkqK7s*o*QD}vi41B2 z3fcEimm^^oP06@H!H7<*OFp$wFNg+IvVYAp>DjdG96^`19#Pj*)JQ>diio-rbx8ARdnl-R?Bb99CsC`~vna+;u z-n5cJ{~Ics#r=fdQ~t7#B$)V>8d#Rmx*wEa3~x90)*QIa=T7fe)euCbKv#jd?Pyp| z{aA76U#^09{WC?747fA~iZjedS^f;}2Pu3eWZEn!XF={A-HfNTX&;fvB!sXbWS#F# zS|x)8XdZIs&2~NrKMIK32UCY5DK8%A(>Y+*XL-Lp#7iw8hcPulHl$63S&LAVD4fG* zDsw6L_D#&9)t?!C0J5Koc7a>LY}Wa}ApoLAh4QTkGe>CwJlcw6tA{pb<=P2Gu2eTy z!_zVh&D7j-133mm1(VBB3n7mrvhKT)OyV~PW-+`3Fj)4m<%AS>4-}k=g~!R~HZw^2 zC1^~WKGV+7JpO;Zlo6tgy*mN~`;5 zO_4PdtVetXIB8sS99K30g>4x{(ofe!?4lSsFU6PNt#4R35zz0#MuJsl#U0UzCAq=L z)ACoy>3NU|O0%<}SR|MoPorpjM3jYX>*2vDEBYW^Mgr-rakYZMi8hZK{iW5kG0h=* zm){J_6@JbV-^G7k&es++hWkQxn?+srx^%ngHzWYrbWzr+`=(q6N9Dei8>T=KN|K-w^rA5H+2A!InyP{r`W0IO5&=^O zhaWcO+>?S6qWHt&%1A?&1PP6pGlR%2j?&gEEXkbV%X&7JJAaD1AwlK6Ln7|fsmYOp zns$i9Nl_Xes2i}L4NF-q0dOsd`-tHT-q*tD`9@=+BtGD5$7cuY7rj)o1dJ#?Y~mXT#aqU2*?C@j7i6M(F|5$?Pw}7fyyxBp=L-A=qUQ z{&7$`do53aWDik=j>=l@hZh!h`#aE97~JpkX1>hlnS8sBXfS<#Tk%>h<~ zIQ1g)IiG0>WH0-fx$mNw7>YavJuUQW^9S_`L5*O|S5raj&cw*Zp7?*{X_s*(-AXQh7}AmA0Mx`Vf7p%P78*2IRcpc-%2%3FHVC z_2r-=TxHggUHKnguGi37cGzm_NSyWCQ(`q6`vk2v4WPPeRQ;TR3E67X@CL1->#fl|x0OPcf zEF4n?S8LLa86r%zk10dYP@8KsRk zAB6T!3Ir1zXBNrZZ+{|>HwxKSLaSZSG3w(;w7TRMFG&|EQ5oc@yPu>PstCwPV=hJ6 zv7iF)l7uhQ>sG@FDK}mnkEz44&B|1Zc2 zPkv5$rwzI&QQp96aIvks*dsBO^?V?JoWyK9Mu)BcSP5^8VuFLLa)Qe92IvZ;sUU*%l~IJn67T}afuEu}4cJC7+wBI-+Xc*adiaBIgly@C)aP-uKFVG_xmogRU{DL$ zn|W1hLzX=HIn=xn{(MQB$-3#1I<-C7Iw*PRH^-W9{f53A<(b7`I{R9QcEnAI%$hmC z?$;BShSe>xMq)qj=%}$$-EuEddw^*lZvd$CaCkW+PCnkZzLa|-S}-WRUnm-&;ERw5 zPnv^^P%)Z>%;LVnwxN9_9ALrfun4#PlF#bk3vdYf49qB{;{L72uwm7B5#?#|>&o)U zr=?>BlJ}^Nh=|Eg%S*;CJvBirz8ih-Ka;qfp|@@vL%7+fMcJfi?nz@>r3dX$a3jy` zyh7R-OQmp1^az-!6}3>C0ri?nF~h<8uUJrXX&&$vj%QG{4A0I0%?acFw+uaS_BTdbIthbHPUG@~rze86A>9DdpD#XI(& zQf-EkbDfVnD__nZIwoBcBVnlU4%o_bL{aNUP!~na-NsQ{)!0i+2o;xC!QEzc?L@SR z?^8N!Phtp zKNWntz(iQH&$oMK;+NC7j{RjGwCXbmuNi>gj`2fQ=~vAORtjRQc<>49FTcrc&_CWW z!?D&J)=*dKX6<|}xWto)y$;55CVRifMRCf@kecBEDJJQps`Y2ET~)@*uy<&MW6f!H z{Yu29qhDV{OX-h<-ff_GO$HHqP4?E*0ZCz{&9@IS7a2IcMc|RO&>q^ma%XShLDkL~ zMbU3}cI2*>R@m0Mk8OfV1+VwroIY>f_D+OTUuLB_IXV0%K@!(va*w-2) z>%^F?RLD=e=*XZV_|~HMW}{%!cEJG0hy$A=NCJ6YsE@6CNOs2bb+C6ZdI+tux^b%! z=5FsWV}%#QZdB1;tZaWUFrSRqXjYyz({*c}Y;|JzpD@~`FQqO&uOYTVARvEc|v#<+Xk z0Vu#|+Y2+BV%|y*pk~m-n5)}<^}70r6<~}@QF*hCg_hp_q0EL=7xwL~V}FzGS{Nd= zMDNVd6sx5^AOrCfKICO4(a+^)nqhD>L%t>@-LMb|tjIU?V@Ztu5oC`W)YqYbsp1!g z8FQs!URNnN_`B-JF{m7Z{rTzi7~}&j5phV$e!X+?r)nmr*t`JB#Fbqs)66|KYD

8Z!u_< z6OIC$CBG2xh8D=H@X(|Y;ZN3{XLus_q~=F;?C(ePX@YyyYz;pGExhm3Xma0e)rzNQ z-9);$yVRbhwf%mDWuA`L{B}XsnbHt#0Pe z7Eb=7_s{2x`EXQkhO~>enL~))sdvt#LcR9Ujbd-RMdy0t-YIZ-?{xq9simc*^x^+{ z@zU8^TeIc+`S`fJtmUv}*xT`PaS7aiuiUqeLM#JvNftHOYf70O<-gT`?Gm(lPp?zG zatOnYy}Um>^grVg{rL0n@*O=|UQly&Y$IL3-Ea5o&{&_FyH|I-0#$r@RUU>Rv7*W9%hu4%f*@JDArhMp`C>4; zA%P^r*UioMrJnO;sL$HZCzf>gpLsUuPe;>ApBNpOc4h`NX+qlWbhG01<<2EUpAWn; zuCzAKPYGb^m9_7F?9saX4PiFrm4Ztxiq~~_F=y(LRL@si$$75EE&SzQwrom18`JYX z0cNxNp%e}}DlaAPzMn3gJH(2g3ya^B65e(GXzEcqzvr`7wGo}9NKp$35v?vV{xNA? zrn6u9-kEulc1aZHYBnB?oag_t_h(tkzvaYMH(f5e;aqZ?0r?@1eN8@Q8n$qw?`!n> zRZoX<<>sRVXUJmwJ>Ezxmjyl{nu^f{c&t>9YPp9AJ$=P)NrL-up;dOdfJmJv?->YG z-*(&tLV)l!;^!g11{|V6wM)zO%XDA?LEZ9xJUc-Uu(1mHO>l)_Pyr_)4ToPF-5lSt zB_U#Ax31SIuJSQ4iAD<14yOE~O@*xbw?s`gABOBBsL|^73|1O#FuYD|Vs6r{?nSjI zjDQR5z{}kJB4w@eb@P*3IT$8Iv);kEi0SVitMIKJvWKEMV$#mTmnn7rb~BZsb|!nJ z43(m(wrD~WYsxg1DQ3$;9PAVPJByUXMO@$4$2 znOACU!gM^=`UOFPNU&wVU;xS3KwKXPA2d>ZZyqC*x;5IJBlA7PrQ`}@NVo5HH1)-d zCWS8xjpiJ7F_W}m+YZlZbkJ7lfeMJ5*x~O!m$1|P_cf2))mjG_1qu6DL}`V4u<~-$ zWeICq(>@nq+qb1$u$<{`YHk}_24q^os^0-~rxE#7EKT8eF}M~n3w z^>^dN6qUwRcB6%2MunqgTOx(SMLm)AKg;q43K#*SwN|23PS!~Qu>3y;_=0ueIw*x& z&NVq3cG5Y&1kpq&GLm# zG5eaV2EuE7S0D4xudd&lmvm&bYhKQs#dW~*U7g7ADGKe;uBo_R<=niy2KFp9ED$cZ zYmbfsxOZ1UiAHxc(sbSq9C4IhPJ8{2`i~fc*01%0nU01>zE_T1;gos}3Z*x&CL5{o zV_$TVyqWIbJ`kfu9%K&kZj%O5{B};eZ!wkj=_h6i?F<$y9OG7e+jnmK8r`h7(%X1> zcHmruIKO5Vg5k?Ls(PEm^P(3vjYXR7zHUpYNS!W!##c@s{LcDa3tw1>_N2{|{XC)C zZtQ`r_wj)TS*Qu#+7*tw9`=kAsoe@8$)CuF%P<1ThV*WGco~}g=^SQRiuK#e%J!Kb zRN=lFiEPWDbjrDBewDs}UE0It(KAxinJU%~CcQ9biV-49#IIY@PDNGo-bH8A!NfCj z*2MPZ9)BFj>qof*yAhR=nT5g_au}sEzQX=m-$|wJ~Ry3A_nUg!e~QT z`PsSS%mNC?2>Jd71xVfIlY%aUmHq|$h6RcxEJ|<q zrq2eldE#9Yfq>EuW{Wr&kFtF|28VVu8)(GP^RlJqn-(VGTMQ2s9&T$DCFj?r>(24} zKh2kq2dR&+p9EW;d3CjH0UdNdv0n`Edvq6w4n*R*Fcn5E)O|+af%hZYX8<E%;Z4e~!})t*H7v=P%btMN$%v54>L&deLAP8#8cfz1vB?9(Bg#qd zOLo4gE^D{& z+Z-FuwH&Z+08w{;>xObz3}l1}@YdpFq{IK?)+T_AR^Z?PCn7Aw6SDuxo%Y2*FDxdJ ztHs&l^ymhr3_c8<{NBat@APrF0^LfQbefii=1~GtVoZy z4u27?#u}b4N~?U#tjYH2_3v~hbD68K)APNosi8Ze+MMx5RQcKR*n$(|#o1H9#}LZ% zX^x@Gi!T$GvtPEN|BA1e7eV+bhcy-=;>04K+oC42XMm&gmBZni1;H4i&|G>M?6P7 zp9@?6K}E3uO+c}b5XA9L^MrL*R-?*+CpMc?q0+xZtK!dT#B3$Bm-pm30C8tSb~Le= zg3MH5010o?SHo|idktTs8^V&&p&5?$3ss)ioET=9p%=PrIZW;VD_Jy6XuC=j5rvYuL^HRA89Kd@y)VY6OksV&pq`zMOjMP`|I2igRJ!!cC!I4J z5BW)Xos99BcBp~1QAca-^A4theyqL50=dzxJ5+VJCe5i^dZDj23;jHjXx&oM?kZl+b~; zG0oUL??X?^!M?1^qC4HN91Pkw_@lGHXjNcqCycnZlEcks#HAh`%99wg{ejVEe>K@= zs}rJ<$bIQ7-ZOm;ge8=aOZOy3UPtT8Vi@KwlY_^?W@Uht@0V0aJ_$%7&}UV}7z|9y ze`^0xAlqhMsfYC<`esHeqglc8fLV=VCMpUh5Yp;oo`|NFJzS*E&@meC#hwRpZ#hy} zP;&r;B_0EfE!j3{$LI{`5EJly>`j2W26Yn0KAb`&#S6K(20}IEfjcb4e;KW_p*$|) z5l98zZwa?lL!R%+=tq>r(LmbrzfSatxyBoQqO*05BJ0NKSecp|2{`t1cJ*yYawy_f z?PeZlx^zj7rlc=`P8`;OA5ium$^Dz0Nt7W$BTDPerHD!FRFX>h1B;v_;@iV z>HJ;i6#H&MXj_PY9KV;nxqV>&>i*x55_8)p9roeqz8s2vYdqwvqcuq zw@nriRa2 z2V%<*f6YoweZd$Dsmt{J!agUn^3<+M^czqPC-5xW7#R)w#$@PqJ;wnT&Sb=XdFDy9T$ij?Q6VsioPuj zVtI&ri_@NbdnzV;&MmpUjS@z|A=I8&z$V7G0Fe7Cuc;mVW@+r|hq+ns@>aq{hA;*B z@6ka0m)s!JxOoZqmoh~JgqOAE0HMD?{q`e8{7Yl~D`KMTF!z5Ut^cz=ZMuzlqVINn zj7Q$0%}D@ho&VIzQ@iEQp((F5Tu&3e^CE~WDPb#E#h8?D^Z%6J4EREL6O{fRQ${W_ zecRX*8UR2RPm5A6_jWhofs?7ufa5Z`c=is`eyeD4*(tK- zt7tjX59;PSYM1RTR6E;a>!jhy0=tyT@8iQlB$NO$&i^g6(-)At>AB;PhG=ttS)zZh zE-0yyL6~5kjmJq|l=nZi%$d{4U|lrgw$4%-`@eFx@L%sHHCDs=r0w$Ckk6VulPVh^OCs%3{;Y~6r;>8vXY#NN zpa%mgAz40s5KsvLgS-}5E_zPkP8Eo<*&a$^PAz&VHY(XQ^{UJVQe_wkGHhyWI1`(Hrozn&kOuvg#{zyipf4`GkM z{&!n19|8p)W%N!Sh`mifx(55V8A<^W&Bks;+2yClo0Isj-1JIaT@Wb4aDVukoAO>y z4nBqm1XnwlCy`f~)dWzz-JHf$=e-k7SL;;AH8#Dg>;(QGc?|ZbU4WOwH~X6Dnr>;^ zU2eL5l(lyp{O}f5=uckKYi|hU e?EZ{Cqy_^63pxV*6aoSH{B!~^J%SYSfc!7;xvM(> literal 0 HcmV?d00001 diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/Chart.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/Chart.yaml new file mode 100644 index 0000000000..83ca3412ad --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/release-name: rancher-logging-crd +apiVersion: v1 +description: Installs the CRDs for rancher-logging. +name: rancher-logging-crd +type: application +version: 104.2.0+up4.8.0 diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/README.md b/charts/rancher-logging-crd/104.2.0+up4.8.0/README.md new file mode 100644 index 0000000000..d4beb54faf --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/README.md @@ -0,0 +1,2 @@ +# rancher-logging-crd +A Rancher chart that installs the CRDs used by rancher-logging. diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_eventtailers.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_eventtailers.yaml new file mode 100644 index 0000000000..eb2535fcdc --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_eventtailers.yaml @@ -0,0 +1,2467 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: eventtailers.logging-extensions.banzaicloud.io +spec: + group: logging-extensions.banzaicloud.io + names: + kind: EventTailer + listKind: EventTailerList + plural: eventtailers + singular: eventtailer + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + containerOverrides: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + controlNamespace: + type: string + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + positionVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + workloadMetaOverrides: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + workloadOverrides: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + containers: + items: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + required: + - controlNamespace + type: object + status: + type: object + type: object + served: true + storage: true diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_hosttailers.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_hosttailers.yaml new file mode 100644 index 0000000000..d49355f12b --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging-extensions.banzaicloud.io_hosttailers.yaml @@ -0,0 +1,2651 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: hosttailers.logging-extensions.banzaicloud.io +spec: + group: logging-extensions.banzaicloud.io + names: + kind: HostTailer + listKind: HostTailerList + plural: hosttailers + singular: hosttailer + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + enableRecreateWorkloadOnImmutableFieldChange: + type: boolean + fileTailers: + items: + properties: + buffer_chunk_size: + type: string + buffer_max_size: + type: string + containerOverrides: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + disabled: + type: boolean + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + name: + type: string + path: + type: string + read_from_head: + type: boolean + skip_long_lines: + type: string + required: + - name + type: object + type: array + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + systemdTailers: + items: + properties: + containerOverrides: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + disabled: + type: boolean + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + maxEntries: + type: integer + name: + type: string + path: + type: string + systemdFilter: + type: string + required: + - name + type: object + type: array + workloadMetaOverrides: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + workloadOverrides: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + containers: + items: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + command: + items: + type: string + type: array + image: + type: string + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + pullPolicy: + type: string + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + status: + type: object + type: object + served: true + storage: true diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusterflows.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusterflows.yaml new file mode 100644 index 0000000000..2d0586cc66 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusterflows.yaml @@ -0,0 +1,2166 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusterflows.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: ClusterFlow + listKind: ClusterFlowList + plural: clusterflows + singular: clusterflow + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + flowLabel: + type: string + globalOutputRefs: + items: + type: string + type: array + includeLabelInRouter: + type: boolean + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + flowLabel: + type: string + globalOutputRefs: + items: + type: string + type: array + includeLabelInRouter: + type: boolean + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusteroutputs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusteroutputs.yaml new file mode 100644 index 0000000000..4a5f05e03b --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_clusteroutputs.yaml @@ -0,0 +1,14812 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusteroutputs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: ClusterOutput + listKind: ClusterOutputList + plural: clusteroutputs + singular: clusteroutput + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_cloud: + type: string + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + slow_flush_log_threshold: + type: string + required: + - azure_container + - azure_storage_account + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + slow_flush_log_threshold: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + slow_flush_log_threshold: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + enabledNamespaces: + items: + type: string + type: array + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + recompress: + type: boolean + slow_flush_log_threshold: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + slow_flush_log_threshold: + type: string + time_as_integer: + type: boolean + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + transport: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + slow_flush_log_threshold: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + gelf: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + host: + type: string + port: + type: integer + protocol: + type: string + tls: + type: boolean + tls_options: + additionalProperties: + type: string + type: object + required: + - host + - port + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + slow_flush_log_threshold: + type: string + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_id: + type: string + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + discard_kafka_delivery_failed: + type: boolean + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + kafka_agg_max_bytes: + type: integer + kafka_agg_max_messages: + type: integer + keytab: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + principal: + type: string + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + slow_flush_log_threshold: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - brokers + - format + type: object + kinesisFirehose: + properties: + append_new_line: + type: boolean + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + delivery_stream_name: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + required: + - delivery_stream_name + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + hostname: + type: string + ingester_domain: + type: string + ingester_endpoint: + type: string + request_timeout: + type: string + slow_flush_log_threshold: + type: string + tags: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_limit: + type: integer + bulk_limit_warning_limit: + type: integer + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + slow_flush_log_threshold: + type: string + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + include_thread_label: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + remove_keys: + items: + type: string + type: array + slow_flush_log_threshold: + type: string + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + mattermost: + properties: + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + channel_id: + type: string + enable_tls: + type: boolean + message: + type: string + message_color: + type: string + message_title: + type: string + webhook_url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - webhook_url + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + base_uri: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + nullout: + type: object + opensearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + catch_transport_exception_on_retry: + type: boolean + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_opensearch_version: + type: integer + emit_error_for_missing_id: + type: boolean + emit_error_label_event: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + required: + - url + type: object + exception_backup: + type: boolean + fail_on_detecting_os_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + http_backend_excon_nonblock: + type: boolean + id_key: + type: string + ignore_exceptions: + type: string + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_separator: + type: string + log_os_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_os_version: + type: integer + max_retry_putting_template: + type: string + parent_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + routing_key: + type: string + scheme: + type: string + selector_class_name: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_affinity: + type: boolean + target_index_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_exclude_timestamp: + type: boolean + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + truncate_caches_interval: + type: string + unrecoverable_error_types: + type: string + unrecoverable_record_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_os_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + oss: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + access_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - access_key_id + - access_key_secret + - bucket + - endpoint + type: object + protected: + type: boolean + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + slow_flush_log_threshold: + type: string + strftime_format: + type: string + ttl: + type: integer + type: object + relabel: + properties: + label: + type: string + required: + - label + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compress: + properties: + parquet_compression_codec: + type: string + parquet_page_size: + type: string + parquet_row_group_size: + type: string + record_type: + type: string + schema_file: + type: string + schema_type: + type: string + type: object + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + slow_flush_log_threshold: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sqs: + properties: + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + create_queue: + type: boolean + delay_seconds: + type: integer + include_tag: + type: boolean + message_group_id: + type: string + queue_name: + type: string + region: + type: string + slow_flush_log_threshold: + type: string + sqs_url: + type: string + tag_property_name: + type: string + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + slow_flush_log_threshold: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + allow_self_signed_cert: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + enable_system_cert_store: + type: boolean + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + fqdn: + type: string + host: + type: string + insecure: + type: boolean + port: + type: integer + private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + slow_flush_log_threshold: + type: string + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify_fqdn: + type: boolean + version: + type: string + required: + - host + type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_cloud: + type: string + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + slow_flush_log_threshold: + type: string + required: + - azure_container + - azure_storage_account + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + slow_flush_log_threshold: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + slow_flush_log_threshold: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + enabledNamespaces: + items: + type: string + type: array + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + recompress: + type: boolean + slow_flush_log_threshold: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + slow_flush_log_threshold: + type: string + time_as_integer: + type: boolean + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + transport: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + slow_flush_log_threshold: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + gelf: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + host: + type: string + port: + type: integer + protocol: + type: string + tls: + type: boolean + tls_options: + additionalProperties: + type: string + type: object + required: + - host + - port + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + slow_flush_log_threshold: + type: string + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_id: + type: string + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + discard_kafka_delivery_failed: + type: boolean + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + kafka_agg_max_bytes: + type: integer + kafka_agg_max_messages: + type: integer + keytab: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + principal: + type: string + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + slow_flush_log_threshold: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - brokers + - format + type: object + kinesisFirehose: + properties: + append_new_line: + type: boolean + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + delivery_stream_name: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + required: + - delivery_stream_name + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + hostname: + type: string + ingester_domain: + type: string + ingester_endpoint: + type: string + request_timeout: + type: string + slow_flush_log_threshold: + type: string + tags: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_limit: + type: integer + bulk_limit_warning_limit: + type: integer + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + slow_flush_log_threshold: + type: string + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + include_thread_label: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + remove_keys: + items: + type: string + type: array + slow_flush_log_threshold: + type: string + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + mattermost: + properties: + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + channel_id: + type: string + enable_tls: + type: boolean + message: + type: string + message_color: + type: string + message_title: + type: string + webhook_url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - webhook_url + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + base_uri: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + nullout: + type: object + opensearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + catch_transport_exception_on_retry: + type: boolean + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_opensearch_version: + type: integer + emit_error_for_missing_id: + type: boolean + emit_error_label_event: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + required: + - url + type: object + exception_backup: + type: boolean + fail_on_detecting_os_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + http_backend_excon_nonblock: + type: boolean + id_key: + type: string + ignore_exceptions: + type: string + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_separator: + type: string + log_os_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_os_version: + type: integer + max_retry_putting_template: + type: string + parent_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + routing_key: + type: string + scheme: + type: string + selector_class_name: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_affinity: + type: boolean + target_index_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_exclude_timestamp: + type: boolean + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + truncate_caches_interval: + type: string + unrecoverable_error_types: + type: string + unrecoverable_record_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_os_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + oss: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + access_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - access_key_id + - access_key_secret + - bucket + - endpoint + type: object + protected: + type: boolean + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + slow_flush_log_threshold: + type: string + strftime_format: + type: string + ttl: + type: integer + type: object + relabel: + properties: + label: + type: string + required: + - label + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compress: + properties: + parquet_compression_codec: + type: string + parquet_page_size: + type: string + parquet_row_group_size: + type: string + record_type: + type: string + schema_file: + type: string + schema_type: + type: string + type: object + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + slow_flush_log_threshold: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sqs: + properties: + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + create_queue: + type: boolean + delay_seconds: + type: integer + include_tag: + type: boolean + message_group_id: + type: string + queue_name: + type: string + region: + type: string + slow_flush_log_threshold: + type: string + sqs_url: + type: string + tag_property_name: + type: string + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + slow_flush_log_threshold: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + allow_self_signed_cert: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + enable_system_cert_store: + type: boolean + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + fqdn: + type: string + host: + type: string + insecure: + type: boolean + port: + type: integer + private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + slow_flush_log_threshold: + type: string + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify_fqdn: + type: boolean + version: + type: string + required: + - host + type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_flows.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_flows.yaml new file mode 100644 index 0000000000..2a67723623 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_flows.yaml @@ -0,0 +1,2150 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: flows.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Flow + listKind: FlowList + plural: flows + singular: flow + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + flowLabel: + type: string + globalOutputRefs: + items: + type: string + type: array + includeLabelInRouter: + type: boolean + localOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + flowLabel: + type: string + globalOutputRefs: + items: + type: string + type: array + includeLabelInRouter: + type: boolean + localOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespace_labels: + additionalProperties: + type: string + type: object + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentbitagents.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentbitagents.yaml new file mode 100644 index 0000000000..d9148dc802 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentbitagents.yaml @@ -0,0 +1,2535 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: fluentbitagents.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: FluentbitAgent + listKind: FluentbitAgentList + plural: fluentbitagents + singular: fluentbitagent + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + HostNetwork: + type: boolean + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorage: + properties: + storage.backlog.mem_limit: + type: string + storage.checksum: + type: string + storage.delete_irrecoverable_chunks: + type: string + storage.metrics: + type: string + storage.path: + type: string + storage.sync: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + bufferVolumeArgs: + items: + type: string + type: array + bufferVolumeImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + bufferVolumeMetrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + configHotReload: + properties: + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + coroStackSize: + format: int32 + type: integer + customConfigSecret: + type: string + customParsers: + type: string + daemonsetAnnotations: + additionalProperties: + type: string + type: object + disableKubernetesFilter: + type: boolean + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableUpstream: + type: boolean + envVars: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + extraVolumeMounts: + items: + properties: + destination: + pattern: ^/.+$ + type: string + readOnly: + type: boolean + source: + pattern: ^/.+$ + type: string + required: + - destination + - source + type: object + type: array + filterAws: + properties: + Match: + type: string + account_id: + type: boolean + ami_id: + type: boolean + az: + type: boolean + ec2_instance_id: + type: boolean + ec2_instance_type: + type: boolean + hostname: + type: boolean + imds_version: + type: string + private_ip: + type: boolean + vpc_id: + type: boolean + type: object + filterKubernetes: + properties: + Annotations: + type: string + Buffer_Size: + type: string + Cache_Use_Docker_Id: + type: string + DNS_Retries: + type: string + DNS_Wait_Time: + type: string + Dummy_Meta: + type: string + K8S-Logging.Exclude: + type: string + K8S-Logging.Parser: + type: string + Keep_Log: + type: string + Kube_CA_File: + type: string + Kube_CA_Path: + type: string + Kube_Meta_Cache_TTL: + type: string + Kube_Tag_Prefix: + type: string + Kube_Token_File: + type: string + Kube_Token_TTL: + type: string + Kube_URL: + type: string + Kube_meta_preload_cache_dir: + type: string + Kubelet_Port: + type: string + Labels: + type: string + Match: + type: string + Merge_Log: + type: string + Merge_Log_Key: + type: string + Merge_Log_Trim: + type: string + Merge_Parser: + type: string + Regex_Parser: + type: string + Use_Journal: + type: string + Use_Kubelet: + type: string + kube_meta_namespace_cache_ttl: + type: string + namespace_annotations: + type: string + namespace_labels: + type: string + tls.debug: + type: string + tls.verify: + type: string + type: object + filterModify: + items: + properties: + conditions: + items: + properties: + A_key_matches: + properties: + key: + type: string + type: object + Key_does_not_exist: + properties: + key: + type: string + value: + type: string + type: object + Key_exists: + properties: + key: + type: string + type: object + Key_value_does_not_equal: + properties: + key: + type: string + value: + type: string + type: object + Key_value_does_not_match: + properties: + key: + type: string + value: + type: string + type: object + Key_value_equals: + properties: + key: + type: string + value: + type: string + type: object + Key_value_matches: + properties: + key: + type: string + value: + type: string + type: object + Matching_keys_do_not_have_matching_values: + properties: + key: + type: string + value: + type: string + type: object + Matching_keys_have_matching_values: + properties: + key: + type: string + value: + type: string + type: object + No_key_matches: + properties: + key: + type: string + type: object + type: object + type: array + rules: + items: + properties: + Add: + properties: + key: + type: string + value: + type: string + type: object + Copy: + properties: + key: + type: string + value: + type: string + type: object + Hard_copy: + properties: + key: + type: string + value: + type: string + type: object + Hard_rename: + properties: + key: + type: string + value: + type: string + type: object + Remove: + properties: + key: + type: string + type: object + Remove_regex: + properties: + key: + type: string + type: object + Remove_wildcard: + properties: + key: + type: string + type: object + Rename: + properties: + key: + type: string + value: + type: string + type: object + Set: + properties: + key: + type: string + value: + type: string + type: object + type: object + type: array + type: object + type: array + flush: + format: int32 + type: integer + forwardOptions: + properties: + Require_ack_response: + type: boolean + Retry_Limit: + type: string + Send_options: + type: boolean + Tag: + type: string + Time_as_Integer: + type: boolean + Workers: + type: integer + storage.total_limit_size: + type: string + type: object + grace: + format: int32 + type: integer + healthCheck: + properties: + hcErrorsCount: + type: integer + hcPeriod: + type: integer + hcRetryFailureCount: + type: integer + type: object + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + inputTail: + properties: + Buffer_Chunk_Size: + type: string + Buffer_Max_Size: + type: string + DB: + type: string + DB.journal_mode: + type: string + DB.locking: + type: boolean + DB_Sync: + type: string + Docker_Mode: + type: string + Docker_Mode_Flush: + type: string + Docker_Mode_Parser: + type: string + Exclude_Path: + type: string + Ignore_Older: + type: string + Key: + type: string + Mem_Buf_Limit: + type: string + Multiline: + type: string + Multiline_Flush: + type: string + Parser: + type: string + Parser_Firstline: + type: string + Parser_N: + items: + type: string + type: array + Path: + type: string + Path_Key: + type: string + Read_From_Head: + type: boolean + Refresh_Interval: + type: string + Rotate_Wait: + type: string + Skip_Long_Lines: + type: string + Tag: + type: string + Tag_Regex: + type: string + multiline.parser: + items: + type: string + type: array + storage.type: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + logLevel: + type: string + loggingRef: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + mountPath: + type: string + network: + properties: + connectTimeout: + format: int32 + type: integer + connectTimeoutLogError: + type: boolean + dnsMode: + type: string + dnsPreferIpv4: + type: boolean + dnsResolver: + type: string + keepalive: + type: boolean + keepaliveIdleTimeout: + format: int32 + type: integer + keepaliveMaxRecycle: + format: int32 + type: integer + maxWorkerConnections: + type: integer + sourceAddress: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + parser: + type: string + podPriorityClassName: + type: string + position_db: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + positiondb: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + syslogng_output: + properties: + Retry_Limit: + type: string + Workers: + type: integer + json_date_format: + type: string + json_date_key: + type: string + type: object + targetHost: + type: string + targetPort: + format: int32 + type: integer + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentdconfigs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentdconfigs.yaml new file mode 100644 index 0000000000..68b148cc33 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_fluentdconfigs.yaml @@ -0,0 +1,3240 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: fluentdconfigs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: FluentdConfig + listKind: FluentdConfigList + plural: fluentdconfigs + singular: fluentdconfig + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the fluentd configuration active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + bufferVolumeArgs: + items: + type: string + type: array + bufferVolumeImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + bufferVolumeMetrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + compressConfigFile: + type: boolean + configCheck: + properties: + labels: + additionalProperties: + type: string + type: object + strategy: + type: string + timeoutSeconds: + type: integer + type: object + configCheckAnnotations: + additionalProperties: + type: string + type: object + configCheckResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + configReloaderImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + configReloaderResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + disablePvc: + type: boolean + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableMsgpackTimeSupport: + type: boolean + envVars: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + extraArgs: + items: + type: string + type: array + extraVolumes: + items: + properties: + containerName: + type: string + path: + type: string + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeName: + type: string + type: object + type: array + fluentLogDestination: + type: string + fluentOutLogrotate: + properties: + age: + type: string + enabled: + type: boolean + path: + type: string + size: + type: string + required: + - enabled + type: object + fluentdPvcSpec: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + forwardInputConfig: + properties: + add_tag_prefix: + type: string + bind: + type: string + chunk_size_limit: + type: string + chunk_size_warn_limit: + type: string + deny_keepalive: + type: boolean + linger_timeout: + type: integer + port: + type: string + resolve_hostname: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_keepalive_packet: + type: boolean + skip_invalid_event: + type: boolean + source_address_key: + type: string + sourceHostnameKey: + type: string + tag: + type: string + transport: + properties: + ca_cert_path: + type: string + ca_path: + type: string + ca_private_key_passphrase: + type: string + ca_private_key_path: + type: string + cert_path: + type: string + ciphers: + type: string + client_cert_auth: + type: boolean + insecure: + type: boolean + private_key_passphrase: + type: string + private_key_path: + type: string + protocol: + type: string + version: + type: string + type: object + type: object + ignoreRepeatedLogInterval: + type: string + ignoreSameLogInterval: + type: string + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + logFormat: + type: string + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + pdb: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + unhealthyPodEvictionPolicy: + type: string + type: object + podPriorityClassName: + type: string + port: + format: int32 + type: integer + readinessDefaultCheck: + properties: + bufferFileNumber: + type: boolean + bufferFileNumberMax: + format: int32 + type: integer + bufferFreeSpace: + type: boolean + bufferFreeSpaceThreshold: + format: int32 + type: integer + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + rootDir: + type: string + scaling: + properties: + drain: + properties: + annotations: + additionalProperties: + type: string + type: object + deleteVolume: + type: boolean + enabled: + type: boolean + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + pauseImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + type: object + podManagementPolicy: + type: string + replicas: + type: integer + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + sidecarContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + statefulsetAnnotations: + additionalProperties: + type: string + type: object + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumeModImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + volumeMountChmod: + type: boolean + workers: + format: int32 + type: integer + type: object + status: + properties: + active: + type: boolean + logging: + type: string + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggingroutes.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggingroutes.yaml new file mode 100644 index 0000000000..2b465f2d6f --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggingroutes.yaml @@ -0,0 +1,106 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: loggingroutes.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: LoggingRoute + listKind: LoggingRouteList + plural: loggingroutes + shortNames: + - lr + singular: loggingroute + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + - description: Number of notices + jsonPath: .status.noticesCount + name: Notices + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + source: + type: string + targets: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - source + - targets + type: object + status: + properties: + notices: + items: + type: string + type: array + noticesCount: + type: integer + problems: + items: + type: string + type: array + problemsCount: + type: integer + tenants: + items: + properties: + name: + type: string + namespaces: + items: + type: string + type: array + required: + - name + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggings.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggings.yaml new file mode 100644 index 0000000000..46c7f5f6c3 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_loggings.yaml @@ -0,0 +1,20601 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: loggings.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Logging + listKind: LoggingList + plural: loggings + singular: logging + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + status: + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Logging reference + jsonPath: .spec.loggingRef + name: LoggingRef + type: string + - description: Control namespace + jsonPath: .spec.controlNamespace + name: ControlNamespace + type: string + - description: Watched namespaces + jsonPath: .status.watchNamespaces + name: WatchNamespaces + type: string + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + allowClusterResourcesFromAllNamespaces: + type: boolean + clusterDomain: + type: string + configCheck: + properties: + labels: + additionalProperties: + type: string + type: object + strategy: + type: string + timeoutSeconds: + type: integer + type: object + controlNamespace: + type: string + defaultFlow: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + flowLabel: + type: string + globalOutputRefs: + items: + type: string + type: array + includeLabelInRouter: + type: boolean + outputRefs: + items: + type: string + type: array + type: object + enableRecreateWorkloadOnImmutableFieldChange: + type: boolean + errorOutputRef: + type: string + flowConfigCheckDisabled: + type: boolean + flowConfigOverride: + type: string + fluentbit: + properties: + HostNetwork: + type: boolean + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorage: + properties: + storage.backlog.mem_limit: + type: string + storage.checksum: + type: string + storage.delete_irrecoverable_chunks: + type: string + storage.metrics: + type: string + storage.path: + type: string + storage.sync: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + bufferVolumeArgs: + items: + type: string + type: array + bufferVolumeImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + bufferVolumeMetrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + configHotReload: + properties: + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + coroStackSize: + format: int32 + type: integer + customConfigSecret: + type: string + customParsers: + type: string + daemonsetAnnotations: + additionalProperties: + type: string + type: object + disableKubernetesFilter: + type: boolean + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableUpstream: + type: boolean + envVars: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + extraVolumeMounts: + items: + properties: + destination: + pattern: ^/.+$ + type: string + readOnly: + type: boolean + source: + pattern: ^/.+$ + type: string + required: + - destination + - source + type: object + type: array + filterAws: + properties: + Match: + type: string + account_id: + type: boolean + ami_id: + type: boolean + az: + type: boolean + ec2_instance_id: + type: boolean + ec2_instance_type: + type: boolean + hostname: + type: boolean + imds_version: + type: string + private_ip: + type: boolean + vpc_id: + type: boolean + type: object + filterKubernetes: + properties: + Annotations: + type: string + Buffer_Size: + type: string + Cache_Use_Docker_Id: + type: string + DNS_Retries: + type: string + DNS_Wait_Time: + type: string + Dummy_Meta: + type: string + K8S-Logging.Exclude: + type: string + K8S-Logging.Parser: + type: string + Keep_Log: + type: string + Kube_CA_File: + type: string + Kube_CA_Path: + type: string + Kube_Meta_Cache_TTL: + type: string + Kube_Tag_Prefix: + type: string + Kube_Token_File: + type: string + Kube_Token_TTL: + type: string + Kube_URL: + type: string + Kube_meta_preload_cache_dir: + type: string + Kubelet_Port: + type: string + Labels: + type: string + Match: + type: string + Merge_Log: + type: string + Merge_Log_Key: + type: string + Merge_Log_Trim: + type: string + Merge_Parser: + type: string + Regex_Parser: + type: string + Use_Journal: + type: string + Use_Kubelet: + type: string + kube_meta_namespace_cache_ttl: + type: string + namespace_annotations: + type: string + namespace_labels: + type: string + tls.debug: + type: string + tls.verify: + type: string + type: object + filterModify: + items: + properties: + conditions: + items: + properties: + A_key_matches: + properties: + key: + type: string + type: object + Key_does_not_exist: + properties: + key: + type: string + value: + type: string + type: object + Key_exists: + properties: + key: + type: string + type: object + Key_value_does_not_equal: + properties: + key: + type: string + value: + type: string + type: object + Key_value_does_not_match: + properties: + key: + type: string + value: + type: string + type: object + Key_value_equals: + properties: + key: + type: string + value: + type: string + type: object + Key_value_matches: + properties: + key: + type: string + value: + type: string + type: object + Matching_keys_do_not_have_matching_values: + properties: + key: + type: string + value: + type: string + type: object + Matching_keys_have_matching_values: + properties: + key: + type: string + value: + type: string + type: object + No_key_matches: + properties: + key: + type: string + type: object + type: object + type: array + rules: + items: + properties: + Add: + properties: + key: + type: string + value: + type: string + type: object + Copy: + properties: + key: + type: string + value: + type: string + type: object + Hard_copy: + properties: + key: + type: string + value: + type: string + type: object + Hard_rename: + properties: + key: + type: string + value: + type: string + type: object + Remove: + properties: + key: + type: string + type: object + Remove_regex: + properties: + key: + type: string + type: object + Remove_wildcard: + properties: + key: + type: string + type: object + Rename: + properties: + key: + type: string + value: + type: string + type: object + Set: + properties: + key: + type: string + value: + type: string + type: object + type: object + type: array + type: object + type: array + flush: + format: int32 + type: integer + forwardOptions: + properties: + Require_ack_response: + type: boolean + Retry_Limit: + type: string + Send_options: + type: boolean + Tag: + type: string + Time_as_Integer: + type: boolean + Workers: + type: integer + storage.total_limit_size: + type: string + type: object + grace: + format: int32 + type: integer + healthCheck: + properties: + hcErrorsCount: + type: integer + hcPeriod: + type: integer + hcRetryFailureCount: + type: integer + type: object + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + inputTail: + properties: + Buffer_Chunk_Size: + type: string + Buffer_Max_Size: + type: string + DB: + type: string + DB.journal_mode: + type: string + DB.locking: + type: boolean + DB_Sync: + type: string + Docker_Mode: + type: string + Docker_Mode_Flush: + type: string + Docker_Mode_Parser: + type: string + Exclude_Path: + type: string + Ignore_Older: + type: string + Key: + type: string + Mem_Buf_Limit: + type: string + Multiline: + type: string + Multiline_Flush: + type: string + Parser: + type: string + Parser_Firstline: + type: string + Parser_N: + items: + type: string + type: array + Path: + type: string + Path_Key: + type: string + Read_From_Head: + type: boolean + Refresh_Interval: + type: string + Rotate_Wait: + type: string + Skip_Long_Lines: + type: string + Tag: + type: string + Tag_Regex: + type: string + multiline.parser: + items: + type: string + type: array + storage.type: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + logLevel: + type: string + loggingRef: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + mountPath: + type: string + network: + properties: + connectTimeout: + format: int32 + type: integer + connectTimeoutLogError: + type: boolean + dnsMode: + type: string + dnsPreferIpv4: + type: boolean + dnsResolver: + type: string + keepalive: + type: boolean + keepaliveIdleTimeout: + format: int32 + type: integer + keepaliveMaxRecycle: + format: int32 + type: integer + maxWorkerConnections: + type: integer + sourceAddress: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + parser: + type: string + podPriorityClassName: + type: string + position_db: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + positiondb: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + syslogng_output: + properties: + Retry_Limit: + type: string + Workers: + type: integer + json_date_format: + type: string + json_date_key: + type: string + type: object + targetHost: + type: string + targetPort: + format: int32 + type: integer + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + fluentd: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + bufferVolumeArgs: + items: + type: string + type: array + bufferVolumeImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + bufferVolumeMetrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + compressConfigFile: + type: boolean + configCheck: + properties: + labels: + additionalProperties: + type: string + type: object + strategy: + type: string + timeoutSeconds: + type: integer + type: object + configCheckAnnotations: + additionalProperties: + type: string + type: object + configCheckResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + configReloaderImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + configReloaderResources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + disablePvc: + type: boolean + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableMsgpackTimeSupport: + type: boolean + envVars: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + extraArgs: + items: + type: string + type: array + extraVolumes: + items: + properties: + containerName: + type: string + path: + type: string + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeName: + type: string + type: object + type: array + fluentLogDestination: + type: string + fluentOutLogrotate: + properties: + age: + type: string + enabled: + type: boolean + path: + type: string + size: + type: string + required: + - enabled + type: object + fluentdPvcSpec: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + forwardInputConfig: + properties: + add_tag_prefix: + type: string + bind: + type: string + chunk_size_limit: + type: string + chunk_size_warn_limit: + type: string + deny_keepalive: + type: boolean + linger_timeout: + type: integer + port: + type: string + resolve_hostname: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_keepalive_packet: + type: boolean + skip_invalid_event: + type: boolean + source_address_key: + type: string + sourceHostnameKey: + type: string + tag: + type: string + transport: + properties: + ca_cert_path: + type: string + ca_path: + type: string + ca_private_key_passphrase: + type: string + ca_private_key_path: + type: string + cert_path: + type: string + ciphers: + type: string + client_cert_auth: + type: boolean + insecure: + type: boolean + private_key_passphrase: + type: string + private_key_path: + type: string + protocol: + type: string + version: + type: string + type: object + type: object + ignoreRepeatedLogInterval: + type: string + ignoreSameLogInterval: + type: string + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + logFormat: + type: string + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + pdb: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + unhealthyPodEvictionPolicy: + type: string + type: object + podPriorityClassName: + type: string + port: + format: int32 + type: integer + readinessDefaultCheck: + properties: + bufferFileNumber: + type: boolean + bufferFileNumberMax: + format: int32 + type: integer + bufferFreeSpace: + type: boolean + bufferFreeSpaceThreshold: + format: int32 + type: integer + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + rootDir: + type: string + scaling: + properties: + drain: + properties: + annotations: + additionalProperties: + type: string + type: object + deleteVolume: + type: boolean + enabled: + type: boolean + image: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + pauseImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + type: object + podManagementPolicy: + type: string + replicas: + type: integer + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + sidecarContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + statefulsetAnnotations: + additionalProperties: + type: string + type: object + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumeModImage: + properties: + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + volumeMountChmod: + type: boolean + workers: + format: int32 + type: integer + type: object + globalFilters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_cri_logtag_key: + type: string + partial_cri_stream_key: + type: string + partial_key: + type: string + partial_metadata_format: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_cri_logtag: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + force_line_breaks: + type: boolean + languages: + items: + type: string + type: array + match_tag: + type: string + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + elasticsearch_genid: + properties: + hash_id_key: + type: string + hash_type: + type: string + include_tag_in_seed: + type: boolean + include_time_in_seed: + type: boolean + record_keys: + type: string + separator: + type: string + use_entire_record: + type: boolean + use_record_as_seed: + type: boolean + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + geoip2_database: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + kube_events_timestamp: + properties: + mapped_time_key: + type: string + timestamp_fields: + items: + type: string + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + keys: + type: string + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + custom_pattern_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + grok_failure_key: + type: string + grok_name_key: + type: string + grok_pattern: + type: string + grok_patterns: + items: + properties: + keep_time_key: + type: boolean + name: + type: string + pattern: + type: string + time_format: + type: string + time_key: + type: string + timezone: + type: string + required: + - pattern + type: object + type: array + keep_time_key: + type: boolean + local_time: + type: boolean + multiline_start_regexp: + type: string + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + match_tag: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object + type: object + type: array + loggingRef: + type: string + nodeAgents: + items: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + type: string + nodeAgentFluentbit: + properties: + bufferStorage: + properties: + storage.backlog.mem_limit: + type: string + storage.checksum: + type: string + storage.delete_irrecoverable_chunks: + type: string + storage.metrics: + type: string + storage.path: + type: string + storage.sync: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + containersPath: + type: string + coroStackSize: + format: int32 + type: integer + customConfigSecret: + type: string + daemonSet: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + updateStrategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + type: object + disableKubernetesFilter: + type: boolean + enableUpstream: + type: boolean + enabled: + type: boolean + extraVolumeMounts: + items: + properties: + destination: + pattern: ^/.+$ + type: string + readOnly: + type: boolean + source: + pattern: ^/.+$ + type: string + required: + - destination + - source + type: object + type: array + filterAws: + properties: + Match: + type: string + account_id: + type: boolean + ami_id: + type: boolean + az: + type: boolean + ec2_instance_id: + type: boolean + ec2_instance_type: + type: boolean + hostname: + type: boolean + imds_version: + type: string + private_ip: + type: boolean + vpc_id: + type: boolean + type: object + filterKubernetes: + properties: + Annotations: + type: string + Buffer_Size: + type: string + Cache_Use_Docker_Id: + type: string + DNS_Retries: + type: string + DNS_Wait_Time: + type: string + Dummy_Meta: + type: string + K8S-Logging.Exclude: + type: string + K8S-Logging.Parser: + type: string + Keep_Log: + type: string + Kube_CA_File: + type: string + Kube_CA_Path: + type: string + Kube_Meta_Cache_TTL: + type: string + Kube_Tag_Prefix: + type: string + Kube_Token_File: + type: string + Kube_Token_TTL: + type: string + Kube_URL: + type: string + Kube_meta_preload_cache_dir: + type: string + Kubelet_Port: + type: string + Labels: + type: string + Match: + type: string + Merge_Log: + type: string + Merge_Log_Key: + type: string + Merge_Log_Trim: + type: string + Merge_Parser: + type: string + Regex_Parser: + type: string + Use_Journal: + type: string + Use_Kubelet: + type: string + kube_meta_namespace_cache_ttl: + type: string + namespace_annotations: + type: string + namespace_labels: + type: string + tls.debug: + type: string + tls.verify: + type: string + type: object + flush: + format: int32 + type: integer + forwardOptions: + properties: + Require_ack_response: + type: boolean + Retry_Limit: + type: string + Send_options: + type: boolean + Tag: + type: string + Time_as_Integer: + type: boolean + Workers: + type: integer + storage.total_limit_size: + type: string + type: object + grace: + format: int32 + type: integer + inputTail: + properties: + Buffer_Chunk_Size: + type: string + Buffer_Max_Size: + type: string + DB: + type: string + DB.journal_mode: + type: string + DB.locking: + type: boolean + DB_Sync: + type: string + Docker_Mode: + type: string + Docker_Mode_Flush: + type: string + Docker_Mode_Parser: + type: string + Exclude_Path: + type: string + Ignore_Older: + type: string + Key: + type: string + Mem_Buf_Limit: + type: string + Multiline: + type: string + Multiline_Flush: + type: string + Parser: + type: string + Parser_Firstline: + type: string + Parser_N: + items: + type: string + type: array + Path: + type: string + Path_Key: + type: string + Read_From_Head: + type: boolean + Refresh_Interval: + type: string + Rotate_Wait: + type: string + Skip_Long_Lines: + type: string + Tag: + type: string + Tag_Regex: + type: string + multiline.parser: + items: + type: string + type: array + storage.type: + type: string + type: object + livenessDefaultCheck: + type: boolean + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + metricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + network: + properties: + connectTimeout: + format: int32 + type: integer + connectTimeoutLogError: + type: boolean + dnsMode: + type: string + dnsPreferIpv4: + type: boolean + dnsResolver: + type: string + keepalive: + type: boolean + keepaliveIdleTimeout: + format: int32 + type: integer + keepaliveMaxRecycle: + format: int32 + type: integer + maxWorkerConnections: + type: integer + sourceAddress: + type: string + type: object + podPriorityClassName: + type: string + positiondb: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + targetHost: + type: string + targetPort: + format: int32 + type: integer + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + varLogsPath: + type: string + type: object + profile: + type: string + type: object + type: array + skipInvalidResources: + type: boolean + syslogNG: + properties: + bufferVolumeMetrics: + properties: + interval: + type: string + mount_name: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeMetricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + configCheck: + properties: + labels: + additionalProperties: + type: string + type: object + strategy: + type: string + timeoutSeconds: + type: integer + type: object + configCheckPod: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + globalOptions: + properties: + log_level: + type: string + stats: + properties: + freq: + type: integer + level: + type: integer + type: object + stats_freq: + type: integer + stats_level: + type: integer + type: object + jsonKeyDelim: + type: string + jsonKeyPrefix: + type: string + logIWSize: + type: integer + maxConnections: + type: integer + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + metricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + readinessDefaultCheck: + properties: + bufferFileNumber: + type: boolean + bufferFileNumberMax: + format: int32 + type: integer + bufferFreeSpace: + type: boolean + bufferFreeSpaceThreshold: + format: int32 + type: integer + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + service: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + skipRBACCreate: + type: boolean + sourceDateParser: + properties: + format: + type: string + template: + type: string + type: object + sourceMetrics: + items: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + type: array + statefulSet: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + podManagementPolicy: + type: string + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + serviceName: + type: string + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + updateStrategy: + properties: + rollingUpdate: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + partition: + format: int32 + type: integer + type: object + type: + type: string + type: object + volumeClaimTemplates: + items: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: array + type: object + type: object + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + type: object + watchNamespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + watchNamespaces: + items: + type: string + type: array + required: + - controlNamespace + type: object + status: + properties: + configCheckResults: + additionalProperties: + type: boolean + type: object + fluentdConfigName: + type: string + problems: + items: + type: string + type: array + problemsCount: + type: integer + syslogNGConfigName: + type: string + watchNamespaces: + items: + type: string + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_nodeagents.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_nodeagents.yaml new file mode 100644 index 0000000000..beb6c35389 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_nodeagents.yaml @@ -0,0 +1,4788 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: nodeagents.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: NodeAgent + listKind: NodeAgentList + plural: nodeagents + singular: nodeagent + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + loggingRef: + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeAgentFluentbit: + properties: + bufferStorage: + properties: + storage.backlog.mem_limit: + type: string + storage.checksum: + type: string + storage.delete_irrecoverable_chunks: + type: string + storage.metrics: + type: string + storage.path: + type: string + storage.sync: + type: string + type: object + bufferStorageVolume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + containersPath: + type: string + coroStackSize: + format: int32 + type: integer + customConfigSecret: + type: string + daemonSet: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + updateStrategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + type: object + disableKubernetesFilter: + type: boolean + enableUpstream: + type: boolean + enabled: + type: boolean + extraVolumeMounts: + items: + properties: + destination: + pattern: ^/.+$ + type: string + readOnly: + type: boolean + source: + pattern: ^/.+$ + type: string + required: + - destination + - source + type: object + type: array + filterAws: + properties: + Match: + type: string + account_id: + type: boolean + ami_id: + type: boolean + az: + type: boolean + ec2_instance_id: + type: boolean + ec2_instance_type: + type: boolean + hostname: + type: boolean + imds_version: + type: string + private_ip: + type: boolean + vpc_id: + type: boolean + type: object + filterKubernetes: + properties: + Annotations: + type: string + Buffer_Size: + type: string + Cache_Use_Docker_Id: + type: string + DNS_Retries: + type: string + DNS_Wait_Time: + type: string + Dummy_Meta: + type: string + K8S-Logging.Exclude: + type: string + K8S-Logging.Parser: + type: string + Keep_Log: + type: string + Kube_CA_File: + type: string + Kube_CA_Path: + type: string + Kube_Meta_Cache_TTL: + type: string + Kube_Tag_Prefix: + type: string + Kube_Token_File: + type: string + Kube_Token_TTL: + type: string + Kube_URL: + type: string + Kube_meta_preload_cache_dir: + type: string + Kubelet_Port: + type: string + Labels: + type: string + Match: + type: string + Merge_Log: + type: string + Merge_Log_Key: + type: string + Merge_Log_Trim: + type: string + Merge_Parser: + type: string + Regex_Parser: + type: string + Use_Journal: + type: string + Use_Kubelet: + type: string + kube_meta_namespace_cache_ttl: + type: string + namespace_annotations: + type: string + namespace_labels: + type: string + tls.debug: + type: string + tls.verify: + type: string + type: object + flush: + format: int32 + type: integer + forwardOptions: + properties: + Require_ack_response: + type: boolean + Retry_Limit: + type: string + Send_options: + type: boolean + Tag: + type: string + Time_as_Integer: + type: boolean + Workers: + type: integer + storage.total_limit_size: + type: string + type: object + grace: + format: int32 + type: integer + inputTail: + properties: + Buffer_Chunk_Size: + type: string + Buffer_Max_Size: + type: string + DB: + type: string + DB.journal_mode: + type: string + DB.locking: + type: boolean + DB_Sync: + type: string + Docker_Mode: + type: string + Docker_Mode_Flush: + type: string + Docker_Mode_Parser: + type: string + Exclude_Path: + type: string + Ignore_Older: + type: string + Key: + type: string + Mem_Buf_Limit: + type: string + Multiline: + type: string + Multiline_Flush: + type: string + Parser: + type: string + Parser_Firstline: + type: string + Parser_N: + items: + type: string + type: array + Path: + type: string + Path_Key: + type: string + Read_From_Head: + type: boolean + Refresh_Interval: + type: string + Rotate_Wait: + type: string + Skip_Long_Lines: + type: string + Tag: + type: string + Tag_Regex: + type: string + multiline.parser: + items: + type: string + type: array + storage.type: + type: string + type: object + livenessDefaultCheck: + type: boolean + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + metricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + network: + properties: + connectTimeout: + format: int32 + type: integer + connectTimeoutLogError: + type: boolean + dnsMode: + type: string + dnsPreferIpv4: + type: boolean + dnsResolver: + type: string + keepalive: + type: boolean + keepaliveIdleTimeout: + format: int32 + type: integer + keepaliveMaxRecycle: + format: int32 + type: integer + maxWorkerConnections: + type: integer + sourceAddress: + type: string + type: object + podPriorityClassName: + type: string + positiondb: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + type: object + security: + properties: + podSecurityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + targetHost: + type: string + targetPort: + format: int32 + type: integer + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + varLogsPath: + type: string + type: object + profile: + type: string + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_outputs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_outputs.yaml new file mode 100644 index 0000000000..222995229d --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_outputs.yaml @@ -0,0 +1,14078 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: outputs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Output + listKind: OutputList + plural: outputs + singular: output + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_cloud: + type: string + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + slow_flush_log_threshold: + type: string + required: + - azure_container + - azure_storage_account + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + slow_flush_log_threshold: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + slow_flush_log_threshold: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + recompress: + type: boolean + slow_flush_log_threshold: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + slow_flush_log_threshold: + type: string + time_as_integer: + type: boolean + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + transport: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + slow_flush_log_threshold: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + gelf: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + host: + type: string + port: + type: integer + protocol: + type: string + tls: + type: boolean + tls_options: + additionalProperties: + type: string + type: object + required: + - host + - port + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + slow_flush_log_threshold: + type: string + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_id: + type: string + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + discard_kafka_delivery_failed: + type: boolean + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + kafka_agg_max_bytes: + type: integer + kafka_agg_max_messages: + type: integer + keytab: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + principal: + type: string + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + slow_flush_log_threshold: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - brokers + - format + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + hostname: + type: string + ingester_domain: + type: string + ingester_endpoint: + type: string + request_timeout: + type: string + slow_flush_log_threshold: + type: string + tags: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_limit: + type: integer + bulk_limit_warning_limit: + type: integer + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + slow_flush_log_threshold: + type: string + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + include_thread_label: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + remove_keys: + items: + type: string + type: array + slow_flush_log_threshold: + type: string + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + base_uri: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + nullout: + type: object + opensearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + catch_transport_exception_on_retry: + type: boolean + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_opensearch_version: + type: integer + emit_error_for_missing_id: + type: boolean + emit_error_label_event: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + required: + - url + type: object + exception_backup: + type: boolean + fail_on_detecting_os_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + http_backend_excon_nonblock: + type: boolean + id_key: + type: string + ignore_exceptions: + type: string + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_separator: + type: string + log_os_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_os_version: + type: integer + max_retry_putting_template: + type: string + parent_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + routing_key: + type: string + scheme: + type: string + selector_class_name: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_affinity: + type: boolean + target_index_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_exclude_timestamp: + type: boolean + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + truncate_caches_interval: + type: string + unrecoverable_error_types: + type: string + unrecoverable_record_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_os_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + oss: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + access_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - access_key_id + - access_key_secret + - bucket + - endpoint + type: object + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + slow_flush_log_threshold: + type: string + strftime_format: + type: string + ttl: + type: integer + type: object + relabel: + properties: + label: + type: string + required: + - label + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compress: + properties: + parquet_compression_codec: + type: string + parquet_page_size: + type: string + parquet_row_group_size: + type: string + record_type: + type: string + schema_file: + type: string + schema_type: + type: string + type: object + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + slow_flush_log_threshold: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sqs: + properties: + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + create_queue: + type: boolean + delay_seconds: + type: integer + include_tag: + type: boolean + message_group_id: + type: string + queue_name: + type: string + region: + type: string + slow_flush_log_threshold: + type: string + sqs_url: + type: string + tag_property_name: + type: string + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + slow_flush_log_threshold: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + allow_self_signed_cert: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + enable_system_cert_store: + type: boolean + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + fqdn: + type: string + host: + type: string + insecure: + type: boolean + port: + type: integer + private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + slow_flush_log_threshold: + type: string + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify_fqdn: + type: boolean + version: + type: string + required: + - host + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_cloud: + type: string + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + slow_flush_log_threshold: + type: string + required: + - azure_container + - azure_storage_account + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + slow_flush_log_threshold: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + slow_flush_log_threshold: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_ilm_name: + type: string + data_stream_ilm_policy: + type: string + data_stream_ilm_policy_overwrite: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_detecting_es_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys: + type: string + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + recompress: + type: boolean + slow_flush_log_threshold: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + slow_flush_log_threshold: + type: string + time_as_integer: + type: boolean + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + transport: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + slow_flush_log_threshold: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + gelf: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + host: + type: string + port: + type: integer + protocol: + type: string + tls: + type: boolean + tls_options: + additionalProperties: + type: string + type: object + required: + - host + - port + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + slow_flush_log_threshold: + type: string + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_id: + type: string + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + discard_kafka_delivery_failed: + type: boolean + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + kafka_agg_max_bytes: + type: integer + kafka_agg_max_messages: + type: integer + keytab: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + principal: + type: string + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + slow_flush_log_threshold: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - brokers + - format + type: object + kinesisFirehose: + properties: + append_new_line: + type: boolean + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + delivery_stream_name: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + required: + - delivery_stream_name + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + slow_flush_log_threshold: + type: string + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + hostname: + type: string + ingester_domain: + type: string + ingester_endpoint: + type: string + request_timeout: + type: string + slow_flush_log_threshold: + type: string + tags: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_limit: + type: integer + bulk_limit_warning_limit: + type: integer + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + slow_flush_log_threshold: + type: string + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + include_thread_label: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + remove_keys: + items: + type: string + type: array + slow_flush_log_threshold: + type: string + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + mattermost: + properties: + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + channel_id: + type: string + enable_tls: + type: boolean + message: + type: string + message_color: + type: string + message_title: + type: string + webhook_url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - webhook_url + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + base_uri: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + nullout: + type: object + opensearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + catch_transport_exception_on_retry: + type: boolean + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compression_level: + type: string + custom_headers: + type: string + customize_template: + type: string + data_stream_enable: + type: boolean + data_stream_name: + type: string + data_stream_template_name: + type: string + default_opensearch_version: + type: integer + emit_error_for_missing_id: + type: boolean + emit_error_label_event: + type: boolean + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + required: + - url + type: object + exception_backup: + type: boolean + fail_on_detecting_os_version_retry_exceed: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + http_backend_excon_nonblock: + type: boolean + id_key: + type: string + ignore_exceptions: + type: string + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_separator: + type: string + log_os_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_os_version: + type: integer + max_retry_putting_template: + type: string + parent_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + routing_key: + type: string + scheme: + type: string + selector_class_name: + type: string + slow_flush_log_threshold: + type: string + sniffer_class_name: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_affinity: + type: boolean + target_index_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_exclude_timestamp: + type: boolean + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + truncate_caches_interval: + type: string + unrecoverable_error_types: + type: string + unrecoverable_record_types: + type: string + use_legacy_template: + type: boolean + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_os_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + oss: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + access_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - access_key_id + - access_key_secret + - bucket + - endpoint + type: object + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + port: + type: integer + slow_flush_log_threshold: + type: string + strftime_format: + type: string + ttl: + type: integer + type: object + relabel: + properties: + label: + type: string + required: + - label + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compress: + properties: + parquet_compression_codec: + type: string + parquet_page_size: + type: string + parquet_row_group_size: + type: string + record_type: + type: string + schema_file: + type: string + schema_type: + type: string + type: object + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + slow_flush_log_threshold: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + slow_flush_log_threshold: + type: string + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sqs: + properties: + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + create_queue: + type: boolean + delay_seconds: + type: integer + include_tag: + type: boolean + message_group_id: + type: string + queue_name: + type: string + region: + type: string + slow_flush_log_threshold: + type: string + sqs_url: + type: string + tag_property_name: + type: string + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + slow_flush_log_threshold: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + allow_self_signed_cert: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + enable_system_cert_store: + type: boolean + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + fqdn: + type: string + host: + type: string + insecure: + type: boolean + port: + type: integer + private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + slow_flush_log_threshold: + type: string + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify_fqdn: + type: boolean + version: + type: string + required: + - host + type: object + vmwareLogInsight: + properties: + agent_id: + type: string + authentication: + enum: + - nil + - basic + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + config_param: + additionalProperties: + type: string + type: object + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + http_conn_debug: + type: boolean + http_method: + enum: + - post + type: string + log_text_keys: + items: + type: string + type: array + max_batch_size: + type: integer + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + path: + type: string + port: + type: integer + raise_on_error: + type: boolean + rate_limit_msec: + type: integer + request_retries: + type: integer + request_timeout: + type: integer + scheme: + enum: + - http + - https + type: string + serializer: + enum: + - json + type: string + shorten_keys: + additionalProperties: + type: string + type: object + ssl_verify: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + vmwareLogIntelligence: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + disabled: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint_url: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + properties: + authorization: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + content_type: + type: string + structure: + type: string + required: + - authorization + - content_type + - structure + type: object + http_compress: + type: boolean + verify_ssl: + type: boolean + required: + - endpoint_url + - headers + - verify_ssl + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusterflows.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusterflows.yaml new file mode 100644 index 0000000000..51534c3ed5 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusterflows.yaml @@ -0,0 +1,360 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: syslogngclusterflows.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: SyslogNGClusterFlow + listKind: SyslogNGClusterFlowList + plural: syslogngclusterflows + singular: syslogngclusterflow + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + id: + type: string + match: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + parser: + properties: + metrics-probe: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + regexp: + properties: + flags: + items: + type: string + type: array + patterns: + items: + type: string + type: array + prefix: + type: string + template: + type: string + required: + - patterns + type: object + syslog-parser: + properties: + flags: + items: + type: string + type: array + type: object + type: object + rewrite: + items: + properties: + group_unset: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + pattern: + type: string + required: + - pattern + type: object + rename: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + newName: + type: string + oldName: + type: string + required: + - newName + - oldName + type: object + set: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + value: + type: string + required: + - field + - value + type: object + subst: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + flags: + items: + type: string + type: array + pattern: + type: string + replace: + type: string + type: + type: string + required: + - field + - pattern + - replace + type: object + unset: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + required: + - field + type: object + type: object + type: array + type: object + type: array + globalOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + outputMetrics: + items: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + type: array + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusteroutputs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusteroutputs.yaml new file mode 100644 index 0000000000..7227853901 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngclusteroutputs.yaml @@ -0,0 +1,2911 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: syslogngclusteroutputs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: SyslogNGClusterOutput + listKind: SyslogNGClusterOutputList + plural: syslogngclusteroutputs + singular: syslogngclusteroutput + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + elasticsearch: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + custom_id: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + index: + type: string + log-fifo-size: + type: integer + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + logstash_suffix: + type: string + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + template: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + type: + type: string + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + enabledNamespaces: + items: + type: string + type: array + file: + properties: + create_dirs: + type: boolean + dir_group: + type: string + dir_owner: + type: string + dir_perm: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + path: + type: string + persist_name: + type: string + template: + type: string + required: + - path + type: object + http: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + log-fifo-size: + type: integer + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + loggingRef: + type: string + loggly: + properties: + close_on_input: + type: boolean + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flags: + items: + type: string + type: array + flush_lines: + type: integer + host: + type: string + persist_name: + type: string + port: + type: integer + so_keepalive: + type: boolean + suppress: + type: integer + tag: + type: string + template: + type: string + template_escape: + type: boolean + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + transport: + type: string + ts_format: + type: string + required: + - token + type: object + logscale: + properties: + attributes: + type: string + batch_bytes: + type: integer + batch_lines: + type: integer + batch_timeout: + type: integer + body: + type: string + content_type: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + extra_headers: + type: string + persist_name: + type: string + rawstring: + type: string + timezone: + type: string + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + loki: + properties: + auth: + properties: + adc: + type: object + alts: + properties: + target-service-accounts: + items: + type: string + type: array + type: object + insecure: + type: object + tls: + properties: + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + type: object + batch-lines: + type: integer + batch-timeout: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + labels: + additionalProperties: + type: string + type: object + log-fifo-size: + type: integer + persist_name: + type: string + retries: + type: integer + template: + type: string + time_reopen: + type: integer + timestamp: + enum: + - current + - received + - msg + type: string + url: + type: string + workers: + type: integer + type: object + mongodb: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + bulk: + type: boolean + bulk_bypass_validation: + type: boolean + bulk_unordered: + type: boolean + collection: + type: string + dir: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + log-fifo-size: + type: integer + persist_name: + type: string + retries: + type: integer + time_reopen: + type: integer + uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + value_pairs: + properties: + exclude: + type: string + key: + type: string + pair: + type: string + scope: + type: string + type: object + write_concern: + enum: + - unacked + - acked + - majority + type: string + required: + - collection + type: object + mqtt: + properties: + address: + type: string + fallback-topic: + type: string + qos: + type: integer + template: + type: string + topic: + type: string + type: object + openobserve: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + log-fifo-size: + type: integer + method: + type: string + organization: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + port: + type: integer + record: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + stream: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + redis: + properties: + auth: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + command: + properties: + string-list: + items: + type: string + type: array + type: object + command_and_arguments: + items: + type: string + type: array + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + host: + type: string + log-fifo-size: + type: integer + persist_name: + type: string + port: + type: integer + retries: + type: integer + throttle: + type: integer + time-reopen: + type: integer + workers: + type: integer + type: object + s3: + properties: + access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + bucket: + type: string + canned_acl: + type: string + chunk_size: + type: integer + compression: + type: boolean + compresslevel: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flush_grace_period: + type: integer + log-fifo-size: + type: integer + max_object_size: + type: integer + max_pending_uploads: + type: integer + object_key: + type: string + object_key_timestamp: + type: string + persist_name: + type: string + region: + type: string + retries: + type: integer + secret_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + storage_class: + type: string + template: + type: string + throttle: + type: integer + upload_threads: + type: integer + url: + type: string + type: object + splunk_hec_event: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + content_type: + type: string + default_index: + type: string + default_source: + type: string + default_sourcetype: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + event: + type: string + extra_headers: + items: + type: string + type: array + extra_queries: + items: + type: string + type: array + fields: + type: string + headers: + items: + type: string + type: array + host: + type: string + index: + type: string + log-fifo-size: + type: integer + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + source: + type: string + sourcetype: + type: string + time: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + sumologic-http: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + collector: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + deployment: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + persist_name: + type: string + time_reopen: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + sumologic-syslog: + properties: + deployment: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + persist_name: + type: string + port: + type: integer + tag: + type: string + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + type: integer + type: object + syslog: + properties: + close_on_input: + type: boolean + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flags: + items: + type: string + type: array + flush_lines: + type: integer + host: + type: string + persist_name: + type: string + port: + type: integer + so_keepalive: + type: boolean + suppress: + type: integer + template: + type: string + template_escape: + type: boolean + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + transport: + type: string + ts_format: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngconfigs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngconfigs.yaml new file mode 100644 index 0000000000..879463dd46 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngconfigs.yaml @@ -0,0 +1,8104 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: syslogngconfigs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: SyslogNGConfig + listKind: SyslogNGConfigList + plural: syslogngconfigs + singular: syslogngconfig + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + bufferVolumeMetrics: + properties: + interval: + type: string + mount_name: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + bufferVolumeMetricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + configCheck: + properties: + labels: + additionalProperties: + type: string + type: object + strategy: + type: string + timeoutSeconds: + type: integer + type: object + configCheckPod: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + globalOptions: + properties: + log_level: + type: string + stats: + properties: + freq: + type: integer + level: + type: integer + type: object + stats_freq: + type: integer + stats_level: + type: integer + type: object + jsonKeyDelim: + type: string + jsonKeyPrefix: + type: string + logIWSize: + type: integer + maxConnections: + type: integer + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + prometheusRules: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + default: replace + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + type: string + type: object + type: array + scheme: + type: string + tlsConfig: + properties: + ca: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + type: string + cert: + properties: + configMap: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + type: string + insecureSkipVerify: + type: boolean + keyFile: + type: string + keySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + type: string + type: object + type: object + timeout: + type: string + type: object + metricsService: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + readinessDefaultCheck: + properties: + bufferFileNumber: + type: boolean + bufferFileNumberMax: + format: int32 + type: integer + bufferFreeSpace: + type: boolean + bufferFreeSpaceThreshold: + format: int32 + type: integer + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + service: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + allocateLoadBalancerNodePorts: + type: boolean + clusterIP: + type: string + clusterIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + internalTrafficPolicy: + type: string + ipFamilies: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + type: string + loadBalancerClass: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + trafficDistribution: + type: string + type: + type: string + type: object + type: object + serviceAccount: + properties: + automountServiceAccountToken: + type: boolean + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + secrets: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + skipRBACCreate: + type: boolean + sourceDateParser: + properties: + format: + type: string + template: + type: string + type: object + sourceMetrics: + items: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + type: array + statefulSet: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + podManagementPolicy: + type: string + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + serviceName: + type: string + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostname: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + type: object + updateStrategy: + properties: + rollingUpdate: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + partition: + format: int32 + type: integer + type: object + type: + type: string + type: object + volumeClaimTemplates: + items: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: array + type: object + type: object + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + type: object + status: + properties: + active: + type: boolean + logging: + type: string + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngflows.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngflows.yaml new file mode 100644 index 0000000000..712fa42628 --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngflows.yaml @@ -0,0 +1,364 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: syslogngflows.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: SyslogNGFlow + listKind: SyslogNGFlowList + plural: syslogngflows + singular: syslogngflow + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the flow active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + id: + type: string + match: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + parser: + properties: + metrics-probe: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + regexp: + properties: + flags: + items: + type: string + type: array + patterns: + items: + type: string + type: array + prefix: + type: string + template: + type: string + required: + - patterns + type: object + syslog-parser: + properties: + flags: + items: + type: string + type: array + type: object + type: object + rewrite: + items: + properties: + group_unset: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + pattern: + type: string + required: + - pattern + type: object + rename: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + newName: + type: string + oldName: + type: string + required: + - newName + - oldName + type: object + set: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + value: + type: string + required: + - field + - value + type: object + subst: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + flags: + items: + type: string + type: array + pattern: + type: string + replace: + type: string + type: + type: string + required: + - field + - pattern + - replace + type: object + unset: + properties: + condition: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + field: + type: string + required: + - field + type: object + type: object + type: array + type: object + type: array + globalOutputRefs: + items: + type: string + type: array + localOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + properties: + and: + x-kubernetes-preserve-unknown-fields: true + not: + x-kubernetes-preserve-unknown-fields: true + or: + x-kubernetes-preserve-unknown-fields: true + regexp: + properties: + flags: + items: + type: string + type: array + pattern: + type: string + template: + type: string + type: + type: string + value: + type: string + required: + - pattern + type: object + type: object + outputMetrics: + items: + properties: + key: + type: string + labels: + additionalProperties: + type: string + type: object + level: + type: integer + type: object + type: array + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngoutputs.yaml b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngoutputs.yaml new file mode 100644 index 0000000000..09b75d417d --- /dev/null +++ b/charts/rancher-logging-crd/104.2.0+up4.8.0/templates/logging.banzaicloud.io_syslogngoutputs.yaml @@ -0,0 +1,2905 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: syslogngoutputs.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: SyslogNGOutput + listKind: SyslogNGOutputList + plural: syslogngoutputs + singular: syslogngoutput + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Is the output active? + jsonPath: .status.active + name: Active + type: boolean + - description: Number of problems + jsonPath: .status.problemsCount + name: Problems + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + elasticsearch: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + custom_id: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + index: + type: string + log-fifo-size: + type: integer + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + logstash_suffix: + type: string + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + template: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + type: + type: string + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + file: + properties: + create_dirs: + type: boolean + dir_group: + type: string + dir_owner: + type: string + dir_perm: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + path: + type: string + persist_name: + type: string + template: + type: string + required: + - path + type: object + http: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + log-fifo-size: + type: integer + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + loggingRef: + type: string + loggly: + properties: + close_on_input: + type: boolean + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flags: + items: + type: string + type: array + flush_lines: + type: integer + host: + type: string + persist_name: + type: string + port: + type: integer + so_keepalive: + type: boolean + suppress: + type: integer + tag: + type: string + template: + type: string + template_escape: + type: boolean + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + transport: + type: string + ts_format: + type: string + required: + - token + type: object + logscale: + properties: + attributes: + type: string + batch_bytes: + type: integer + batch_lines: + type: integer + batch_timeout: + type: integer + body: + type: string + content_type: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + extra_headers: + type: string + persist_name: + type: string + rawstring: + type: string + timezone: + type: string + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + loki: + properties: + auth: + properties: + adc: + type: object + alts: + properties: + target-service-accounts: + items: + type: string + type: array + type: object + insecure: + type: object + tls: + properties: + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + type: object + batch-lines: + type: integer + batch-timeout: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + labels: + additionalProperties: + type: string + type: object + log-fifo-size: + type: integer + persist_name: + type: string + retries: + type: integer + template: + type: string + time_reopen: + type: integer + timestamp: + enum: + - current + - received + - msg + type: string + url: + type: string + workers: + type: integer + type: object + mongodb: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + bulk: + type: boolean + bulk_bypass_validation: + type: boolean + bulk_unordered: + type: boolean + collection: + type: string + dir: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + log-fifo-size: + type: integer + persist_name: + type: string + retries: + type: integer + time_reopen: + type: integer + uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + value_pairs: + properties: + exclude: + type: string + key: + type: string + pair: + type: string + scope: + type: string + type: object + write_concern: + enum: + - unacked + - acked + - majority + type: string + required: + - collection + type: object + mqtt: + properties: + address: + type: string + fallback-topic: + type: string + qos: + type: integer + template: + type: string + topic: + type: string + type: object + openobserve: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + log-fifo-size: + type: integer + method: + type: string + organization: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + port: + type: integer + record: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + stream: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + redis: + properties: + auth: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + command: + properties: + string-list: + items: + type: string + type: array + type: object + command_and_arguments: + items: + type: string + type: array + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + host: + type: string + log-fifo-size: + type: integer + persist_name: + type: string + port: + type: integer + retries: + type: integer + throttle: + type: integer + time-reopen: + type: integer + workers: + type: integer + type: object + s3: + properties: + access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + bucket: + type: string + canned_acl: + type: string + chunk_size: + type: integer + compression: + type: boolean + compresslevel: + type: integer + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flush_grace_period: + type: integer + log-fifo-size: + type: integer + max_object_size: + type: integer + max_pending_uploads: + type: integer + object_key: + type: string + object_key_timestamp: + type: string + persist_name: + type: string + region: + type: string + retries: + type: integer + secret_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + storage_class: + type: string + template: + type: string + throttle: + type: integer + upload_threads: + type: integer + url: + type: string + type: object + splunk_hec_event: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + body-prefix: + type: string + body-suffix: + type: string + content_type: + type: string + default_index: + type: string + default_source: + type: string + default_sourcetype: + type: string + delimiter: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + event: + type: string + extra_headers: + items: + type: string + type: array + extra_queries: + items: + type: string + type: array + fields: + type: string + headers: + items: + type: string + type: array + host: + type: string + index: + type: string + log-fifo-size: + type: integer + method: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + persist_name: + type: string + response-action: + additionalProperties: + type: string + type: object + retries: + type: integer + source: + type: string + sourcetype: + type: string + time: + type: string + time_reopen: + type: integer + timeout: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + url: + type: string + user: + type: string + user-agent: + type: string + workers: + type: integer + type: object + sumologic-http: + properties: + batch-bytes: + type: integer + batch-lines: + type: integer + batch-timeout: + type: integer + body: + type: string + collector: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + deployment: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + headers: + items: + type: string + type: array + persist_name: + type: string + time_reopen: + type: integer + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + url: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + sumologic-syslog: + properties: + deployment: + type: string + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + persist_name: + type: string + port: + type: integer + tag: + type: string + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + token: + type: integer + type: object + syslog: + properties: + close_on_input: + type: boolean + disk_buffer: + properties: + compaction: + type: boolean + dir: + type: string + disk_buf_size: + format: int64 + type: integer + mem_buf_length: + format: int64 + type: integer + mem_buf_size: + format: int64 + type: integer + q_out_size: + format: int64 + type: integer + reliable: + type: boolean + required: + - disk_buf_size + - reliable + type: object + flags: + items: + type: string + type: array + flush_lines: + type: integer + host: + type: string + persist_name: + type: string + port: + type: integer + so_keepalive: + type: boolean + suppress: + type: integer + template: + type: string + template_escape: + type: boolean + tls: + properties: + ca_dir: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cert_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + cipher-suite: + type: string + key_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + peer_verify: + type: boolean + ssl_version: + enum: + - sslv3 + - tlsv1 + - tlsv1_0 + - tlsv1_1 + - tlsv1_2 + - tlsv1_3 + type: string + use-system-cert-store: + type: boolean + type: object + transport: + type: string + ts_format: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/index.yaml b/index.yaml index e15b9e4a63..bb692dc010 100755 --- a/index.yaml +++ b/index.yaml @@ -17739,6 +17739,20 @@ entries: urls: - assets/rancher-logging-crd/rancher-logging-crd-105.0.0+up4.8.0.tgz version: 105.0.0+up4.8.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/release-name: rancher-logging-crd + apiVersion: v1 + created: "2025-02-25T21:43:03.380039406-03:00" + description: Installs the CRDs for rancher-logging. + digest: 16348871b91843d294d8b5d305dc5f78ad9cc5a22f8ffccbaa60ead853bdbac9 + name: rancher-logging-crd + type: application + urls: + - assets/rancher-logging-crd/rancher-logging-crd-104.2.0+up4.8.0.tgz + version: 104.2.0+up4.8.0 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index 3e00b835a6..e15f1733ad 100644 --- a/release.yaml +++ b/release.yaml @@ -2,3 +2,5 @@ rancher-webhook: - 104.0.7+up0.5.7 rancher-logging: - 104.2.0+up4.8.0 +rancher-logging-crd: + - 104.2.0+up4.8.0 From 0b5b65e8956411646d0ff9f61deab41af88d8828 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:44:06 -0300 Subject: [PATCH 05/19] fp: fleet 104.1.5+up0.10.9 --- assets/fleet/fleet-104.1.5+up0.10.9.tgz | Bin 0 -> 6446 bytes charts/fleet/104.1.5+up0.10.9/Chart.yaml | 18 ++ charts/fleet/104.1.5+up0.10.9/README.md | 30 +++ .../104.1.5+up0.10.9/templates/_helpers.tpl | 22 ++ .../104.1.5+up0.10.9/templates/configmap.yaml | 29 ++ .../templates/deployment.yaml | 251 ++++++++++++++++++ .../templates/deployment_gitjob.yaml | 136 ++++++++++ .../job_cleanup_clusterregistrations.yaml | 41 +++ .../templates/job_cleanup_gitrepojobs.yaml | 44 +++ .../104.1.5+up0.10.9/templates/rbac.yaml | 113 ++++++++ .../templates/rbac_gitjob.yaml | 133 ++++++++++ .../104.1.5+up0.10.9/templates/service.yaml | 36 +++ .../templates/service_gitjob.yaml | 51 ++++ .../templates/serviceaccount.yaml | 12 + .../templates/serviceaccount_gitjob.yaml | 6 + charts/fleet/104.1.5+up0.10.9/values.yaml | 121 +++++++++ index.yaml | 22 ++ release.yaml | 2 + 18 files changed, 1067 insertions(+) create mode 100644 assets/fleet/fleet-104.1.5+up0.10.9.tgz create mode 100644 charts/fleet/104.1.5+up0.10.9/Chart.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/README.md create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/_helpers.tpl create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/configmap.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/deployment.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/deployment_gitjob.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_clusterregistrations.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_gitrepojobs.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/rbac.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/rbac_gitjob.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/service.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/service_gitjob.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/serviceaccount.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/templates/serviceaccount_gitjob.yaml create mode 100644 charts/fleet/104.1.5+up0.10.9/values.yaml diff --git a/assets/fleet/fleet-104.1.5+up0.10.9.tgz b/assets/fleet/fleet-104.1.5+up0.10.9.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f0c5c0f2fbc6fc4d6948f4593061bd8251f467d0 GIT binary patch literal 6446 zcmV+}8PVn+iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhS0gvla6b31sAKY;N%CT2Gr-LHBze{go>>wG`~b7roIQJp z++DUc)0R4t8pccb@6XYtukN-D7$(VD^I>RRN+qdOHyli*A=uq_S6{7#o!pCo~9-(oPgcdU)=b2Tb*|O_yvYYBsTRt zxb^i&ME{Y%y599j6s4bS4eT#ofYKKZUpZLBV!cMgtt_kMG}aD3!jAOQD4`Z zC>6=>&W^8njchrj4T(vKO74Vw%;k<9xx5~Gd5qGbb=rHaPHX@F$C1P2b8nzOc(VNe znuIaRrxE~5`2WGd!CsmF@9)3d@c-vXbV|mkIlsV&ORBh-rEJ^oMGrucaTl)Lh66Bd z25}e;cu4(O7mi1#Tnz+DWXki%7@0a=3b$$M-5dHAyx}5Ylpt_Sr4mMfftnyh@i3%5 zygxqyJ~BB!#zPbg6-ww0jzOZhMgh}UDi|V2R4_q--WK#fk!ciS#;MR|w0u789_;PC z)O_*6G=G17(uGE2IW{;2WIhit0JFyuqz-%cM(=n8A&wN_G*UC53aA7kWjw_I)P%}1 zq|1{FT}9F{!vKaeAaH|W2p<{0VQ|=MWy3k_C8Ie)rNtLR2{#i|6BGtXvs54_JPrd; zLL+zBgJH~q5G9NRpPFhsLOpmk1}i;ZNLN`ZE>J^c;B&^%Hve=t$Y*A{r6LMj-WL3z7fGNJGd-;YSUeq~Fs;$&EYOsu z@e`;?ig6eVBnocGObldg0fH(!1BYm6}-VMu0O=|C0ZPNbpZHw_&_3`)N0CcW39@SfbDx z#P@&wHX>5qa1ngp{Wcci_uX&x!hWwo`FuJhATc6>C=4K^(#%=Fr-U*ITmX%L1c5*) zQQCps&@ePZ(Bm?@T=(oTKCl>cc3-vH`rpp$?M^%U^VM#vb6`F?yW5?==&#rI^I*Fr zul?;-LjEE5N$7b)&Xr#ENb^x-WEg6lT5G``z!qFi5K`?mhuRQ9EA75sA`F1Az+h>B zrj(e}w8V%41E9pj@KU9`E#6N<{0`Gobmu2Q!%R(@j#0WI%b*$=o;rZ z%4FWhRETKJJm9|E(L1R*j%k29j0f117-|O2*^alh<>{;olfy+9Ztpb26Ss|Y87M+E zd(sonLoFW=z7qIrOa&HXmND86zV=)#W0AtY{%r;^5R?n5W``k>QqzDNf);(;Ev+6- z%Zti2-ON$NBRRj1yeVoCQdJV*Fdipt)_k1cHHKYidy(J~854y^I{N~Z7H9I3_ly8Z z0Bmi+nX!;a2#16B5JiHIiiFdGf#4&mg{bD(J`eRoVxVg&%3>bNaAv4Wl4}e!dlfU# zb~6t&{2rTWy@CdC&BJ(#C9?Dh)7#G3n8CIoeW%sH5Vd5AHCvl-sRl$%G)Ii$fyT4z zZ2`ERy3jai%ZACZ{T$}Iu~i|3l6+gz?ZS?lMmuKkX@QzW*o7l14B5=i#GzqUo;Rgq zk@H5(OTWVq35%nY4LX7M10Tv_k)cB*0SZXywKzLoXiEF8Q1@pn;iF_&@Ukt~rQ*_m%S3RPf$#@ezd+Ihr^O zd$I(N#GjZs)D<&8299fIfu{4*g~oon;qt5=P?42wBv)v(VkHP5yQYF0mI2p9P`y(m zXu)bR%H%-Atq4cmuFtMDH8jE6^8J4f`sc^*`lpw@ldH4CR56-oSvl`2TuEE1Hd5j6gv+-W=r0BU*6d5U?dd93Fxu>#GNRd>X09P8f!6=Dji@bj ze}X|ACTx&P%tQ3P*>U(KlWYo`Mv=+^w_|F;N7R~I-qox*bix`?W-*mHEF_S*9dfqceC3|KHtf zm-+wxZo9ML|Id*MM?D?yBV|Dsta|^BMA|$e0a2vuTm$I|V5-Y{NwKAzF8pRx%3G&c zHYz!LqYI7OTS&-mf#_+$-Cbil0hYOz?+4-bHW51+gjVL6zPsyI)=*?z-FxPqc6d}Y zF8Q&G>$FkI6MU6QK&Z%0_(v{>|+};-S z1R{oHjEB~;r9ABe?(UKpHtF2b-TnniT_7XR0qYGIFOX}X;rIV7ngminC3D7XE znj+F^&6X+5p$gE@6j)bXmlbPUH&I=ETUu;keMyZjR5!83#U`nPudbi8La|-ZJ6G4Z zSnKXCF+a1Jy}{vx^N;7~BdyvW^W}MB9aA-lhwsl%1_DR)Q;B|xg-P#s|J6sxW%-}; zrL9c?SSZy%1+}dbgf2N*}A=@ zBL`RGcz4${e>ZP$3%R$q$O6sZWQ2a#oG>IFCp{z+y5)`$-PDjMs!;79)vsK`9ELiD z1sL^EY%0ezS=4D@HK;uPjkJv;1bgP4ud)QHQv-8BV;Q`hf{@IQP8wrESy8;mFDgg_nzZUPE8gp$Kd_dU~qo+=IZG9d>)1DjAZAj`W^JokI#-O zNB?Wg70yF#+2fyk&PDN9rBygnslxNHcX@fzzv`d#4=<0;POnb-y^H?U(fjkB{;a}x zrLG=@bneV&6_n5Wm*+n(nBWzb=2=2ogAOe@v+4y=eoKd(&h|x z0d`0Ie_XvgJE}luF%z%c7^(PaW#?lpcQ)3tr(IR3{O$ee(MkX6ynlFhdU$-&Kfn6v z?EJ_6`Nc~6=ANCdX&jl#UO%Fv{@~>7=Xd?n%O@OLH6poDmPePI5!y0P{AV%^D~f*u z$^s4cbx!Tdbzy@T;l*kWp)ZYwlyyT(-XJjUpmMn)ns#5Leb#ux3Ml2H7#s5-D?CKC zgy7;7S}sT-Kl+FZCG%=Zs>uwvP+fT4e%;P3u|?d$TSSW{IEn>%$Y!_|qfM}_QqY>v zY$mE+E;A#ssz&u&L)!$a%!=+dpi<{IR~YOH%&OTmne#c#5U(+;99sdLi65C8Whe_w zL82d<02a;^eo}eO(L?|Y1ihvqjEue6i9+I7S7H&S8r&Db}yts6Dj2+ zu!4CIK8eVXhE!3MMfEihJSyfip?7k!Xm)MX$U76Lm~YyIsm>duaUunNwD6vmYi{_j#47wa5JsB;DOZ;vm}`-8J5QFSW1-RPr#P<=GX`t16nl^|Zp~MGF(R+!a3!QemUCe(gcpSNc94^gfGyl2%_>cPd zpQ|xdANX($VL*%He>&~Gx%Yn#UhTIx@juUzHt|2t5&u(WVfH#|UZHwugpl2&Pa7BH zI%zyC9w^t^rr&uSP?f@Ahu|vA2MU>+e%7v7F(~~pc10tny0Llmszm03bF#Vf!|D^n zPC*f^K0~Zj_yLF*m%!Hy3EO6k# z%1*q8JGbB=4la0%lWPej-Otgr2>lvo*GkA&JG>r>dX3X-G4eHzuNBd+c7ClmfVqH4 zSQfVlD@I&dV7)h+ba-N#?Ivd3<*&u)>|4k|r&Fz3Cip~SjKnPssu^U*RJZ`j5g_9yd(<+sTF zyxPy{Qu{-tvj6V`A6~hLfGc+&B>!IFgWiZ;cK?63eE)BEx6|Id|N1Pc`XZS*yuvUrAl5T-?m?I zG!`Vl`RdI8LsZy|lFRxgQ^vlJXkJ7c;)n}unr!o8BIPV~k3IPITjIf3=2xmY$P%cD zQY9zBr+A$fJ($(zS?XdQTy+u7u3u%G(gPo2D7#<^Z!pl@^9#k^@+aiEcNg4iLh}dY zY3sF|_dfivwqElr_ffBS)*bO@6j<=I&4+e1p<>THOkHf1l^zS_-bz#joFVZ)^3mvo zPO0ibXCthiGA)w-&d~h8hmU(2e(C!^d!_Tg-F9bl|NnWClmCPTRXV`@>zYu1gtbH^ zI24?%q87M5wG5TnL6v*utP735284#QhJA6!kA*3y#Ok71E|&y%D`cj^CwuWAuJJg3pE8; zq%FV#tF8J}tX^|~XV)I!(Hg|kiM^jEYCTRQxQAu|%iUUK4Ff8)45&EMtI{^0`aJIu zibiL*xE_zvHuAv!dsU9I{1-#wulsjkmdO8?`|a}kKYN}1&HI1PlJWxEtUZa@2Vx_5kAW-|bnuVCYkhu!5+9{pyBQec*=uppP3@HmJ8{7P> zMyY5qGSy=l#S^p_^yUG~6J#xH84o|8uM8#H zW)}q|tG;F72QhAMTj%!I(ptGU?7WA}(PKAI6_MVHFn!KNK-prFG-IuR_RtYOJvlrT z9hAcvD=$SnZm+B`E4eEcb%bFV=Stl4nRa#_>8IP{xn3T;%7q&&%lG}CLP@OB|JE1z zZPOC{Z*RA=TYmp>zrDNB|DGe2_}b#YEt@U9Ph zDDB<4oVJp&xoVj8;36K>fD5TqFS5@u=D%(5^+4K_>X`4YZ_)*Y?qHUOCyA z&)SiSsMwX1`fhyFSX5)b!#hmCShc4sDEW;{vK zCoG?-g-up(5&rhza^Z07R~!YzRJ*UK;%A%5fYWr-hSmX@8Ib>YlPo~?sIDCh~I`ugx88Y2X52r?Z ztGM=1(AHpHU4!-7X{*Vx3LC4Qxma7PmWi(2;!1|F20QB!Yxk$nq5sIKtp7VEzE&6T z68*oke^A!{_g-!E|K~~N_^u8`ABoRJCl{`R z=JWG6LW@V%)epI+VR7cyJDn6i9(cJcgiZ5j?#)p}#28iSA<{^2#eE)j;qq{h6AIaN ztm4Jg`gi3E_d-i!GZj|97Fxb|u=tv4sjo(X66T|ff46e3hX1eC1+*;wr@de1{|CD- zH}SvElJ4dIS%EPnx}akfxx2+lD9A*qsF}VUB5LAo}`au_W!Gw?b8159K77$?EmLTTku9x0Q?=p zDI89SP@ZRVsD!svokbFe0_l?wTacCnDCwAC0E)X`JAr8&DjJ2D#I4Gf=WT7lPZMHV zn5O^ZdEX?q7a$u`hTRM($u5!Pe?o<@AoqhQu2}&CJ{qBbSfY?HR7f09 zl`Os;Ar&Q>Kh}+d*o3Nh-sOZ!I~s?n+YEXwNq**2T#p?UCDkRL8my=?KOKZDK;~0D zi4tTyv@_MxJs||#kM$bcwb4BzQ-Xx_KW28#r2H%3KHT6?QicEiYNAvmyE{A9fna_U z97T)rov-a^1_A~C8dF_cdfr=9Ffr>ff*Fq?;OvEh3Asj~s%4kzrSAAcEENzqJUMQ8 zZdwc#+tGVo@~JV(F(Eq|_)1Mspdk}=!(oU%i6z1fg2adjq7Znj;PCuN!iWpd+-ItD zMQ-o3u%r^!rc$7XSeN|CLa6 I0sy1{0EkGX-2eap literal 0 HcmV?d00001 diff --git a/charts/fleet/104.1.5+up0.10.9/Chart.yaml b/charts/fleet/104.1.5+up0.10.9/Chart.yaml new file mode 100644 index 0000000000..5d5d8440ce --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/Chart.yaml @@ -0,0 +1,18 @@ +annotations: + catalog.cattle.io/auto-install: fleet-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/experimental: "true" + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.18.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: fleet +apiVersion: v2 +appVersion: 0.10.9 +description: Fleet Manager - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet +version: 104.1.5+up0.10.9 diff --git a/charts/fleet/104.1.5+up0.10.9/README.md b/charts/fleet/104.1.5+up0.10.9/README.md new file mode 100644 index 0000000000..2f2a4c302a --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/README.md @@ -0,0 +1,30 @@ +# Fleet Helm Chart + +Fleet is GitOps at scale. Fleet is designed to manage multiple clusters. + +## What is Fleet? + +* Cluster engine: Fleet is a container management and deployment engine designed to offer users more control on the local cluster and constant monitoring through GitOps. Fleet focuses not only on the ability to scale, but it also gives users a high degree of control and visibility to monitor exactly what is installed on the cluster. + +* Deployment management: Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, Kustomize, or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy all resources in the cluster. As a result, users can enjoy a high degree of control, consistency, and auditability of their clusters. + +## Introduction + +This chart deploys Fleet on a Kubernetes cluster. It also deploys some of its dependencies as subcharts. + +The documentation is centralized in the [doc website](https://fleet.rancher.io/). + +## Prerequisites + +Get helm if you don't have it. Helm 3 is just a CLI. + + +## Install Fleet + +Install the Fleet Helm charts (there are two because we separate out CRDs for ultimate flexibility.): + +``` +$ helm repo add fleet https://rancher.github.io/fleet-helm-charts/ +$ helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd +$ helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet +``` \ No newline at end of file diff --git a/charts/fleet/104.1.5+up0.10.9/templates/_helpers.tpl b/charts/fleet/104.1.5+up0.10.9/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/104.1.5+up0.10.9/templates/configmap.yaml b/charts/fleet/104.1.5+up0.10.9/templates/configmap.yaml new file mode 100644 index 0000000000..719898d6d8 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fleet-controller +data: + config: | + { + "systemDefaultRegistry": "{{ template "system_default_registry" . }}", + "agentImage": "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}", + "agentImagePullPolicy": "{{ .Values.agentImage.imagePullPolicy }}", + "apiServerURL": "{{.Values.apiServerURL}}", + "apiServerCA": "{{b64enc .Values.apiServerCA}}", + "agentCheckinInterval": "{{.Values.agentCheckinInterval}}", + "agentTLSMode": "{{.Values.agentTLSMode}}", + {{ if .Values.garbageCollectionInterval }} + "garbageCollectionInterval": "{{.Values.garbageCollectionInterval}}", + {{ end }} + "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}}, + "bootstrap": { + "paths": "{{.Values.bootstrap.paths}}", + "repo": "{{.Values.bootstrap.repo}}", + "secret": "{{.Values.bootstrap.secret}}", + "branch": "{{.Values.bootstrap.branch}}", + "namespace": "{{.Values.bootstrap.namespace}}", + "agentNamespace": "{{.Values.bootstrap.agentNamespace}}" + }, + "webhookReceiverURL": "{{.Values.webhookReceiverURL}}", + "githubURLPrefix": "{{.Values.githubURLPrefix}}" + } diff --git a/charts/fleet/104.1.5+up0.10.9/templates/deployment.yaml b/charts/fleet/104.1.5+up0.10.9/templates/deployment.yaml new file mode 100644 index 0000000000..025eb0710a --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/deployment.yaml @@ -0,0 +1,251 @@ +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "fleet-controller{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" +spec: + selector: + matchLabels: + app: fleet-controller + template: + metadata: + labels: + app: fleet-controller + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- if empty $shard.id }} + fleet.cattle.io/shard-default: "true" + {{- end }} + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $.Values.clusterEnqueueDelay }} + - name: FLEET_CLUSTER_ENQUEUE_DELAY + value: {{ $.Values.clusterEnqueueDelay }} + {{- end }} + {{- if $.Values.proxy }} + - name: HTTP_PROXY + value: {{ $.Values.proxy }} + - name: HTTPS_PROXY + value: {{ $.Values.proxy }} + - name: NO_PROXY + value: {{ $.Values.noProxy }} + {{- end }} + {{- if $.Values.cpuPprof }} + - name: FLEET_CPU_PPROF_DIR + value: /tmp/pprof/ + {{- end }} + {{- if $.Values.cpuPprof }} + - name: FLEET_CPU_PPROF_PERIOD + value: {{ quote $.Values.cpuPprof.period }} + {{- end }} + {{- if $.Values.leaderElection.leaseDuration }} + - name: CATTLE_ELECTION_LEASE_DURATION + value: {{$.Values.leaderElection.leaseDuration}} + {{- end }} + {{- if $.Values.leaderElection.retryPeriod }} + - name: CATTLE_ELECTION_RETRY_PERIOD + value: {{$.Values.leaderElection.retryPeriod}} + {{- end }} + {{- if $.Values.leaderElection.renewDeadline }} + - name: CATTLE_ELECTION_RENEW_DEADLINE + value: {{$.Values.leaderElection.renewDeadline}} + {{- end }} + {{- if $.Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- if $.Values.controller.reconciler.workers.bundle }} + - name: BUNDLE_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.bundle }} + {{- end }} + {{- if $.Values.controller.reconciler.workers.bundledeployment }} + - name: BUNDLEDEPLOYMENT_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.bundledeployment }} + {{- end }} +{{- if $.Values.extraEnv }} +{{ toYaml $.Values.extraEnv | indent 8}} +{{- end }} + image: '{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}' + name: fleet-controller + imagePullPolicy: "{{ $.Values.image.imagePullPolicy }}" + {{- if $.Values.metrics.enabled }} + ports: + - containerPort: 8080 + name: metrics + {{- end }} + command: + - fleetcontroller + {{- if $shard.id }} + - --shard-id + - {{ quote $shard.id }} + {{- end }} + {{- if not $.Values.metrics.enabled }} + - --disable-metrics + {{- end }} + {{- if $.Values.debug }} + - --debug + - --debug-level + - {{ quote $.Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /tmp + name: tmp + {{- if $.Values.cpuPprof }} + - mountPath: /tmp/pprof + name: pprof + {{- end }} + {{- if not $shard.id }} # Only deploy cleanup and agent management through sharding-less deployment + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $.Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- if $.Values.leaderElection.leaseDuration }} + - name: CATTLE_ELECTION_LEASE_DURATION + value: {{$.Values.leaderElection.leaseDuration}} + {{- end }} + {{- if $.Values.leaderElection.retryPeriod }} + - name: CATTLE_ELECTION_RETRY_PERIOD + value: {{$.Values.leaderElection.retryPeriod}} + {{- end }} + {{- if $.Values.leaderElection.renewDeadline }} + - name: CATTLE_ELECTION_RENEW_DEADLINE + value: {{$.Values.leaderElection.renewDeadline}} + {{- end }} + {{- if $.Values.controller.reconciler.workers.gitrepo }} + - name: GITREPO_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.gitrepo }} + {{- end }} + {{- if $.Values.controller.reconciler.workers.bundle }} + - name: BUNDLE_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.bundle }} + {{- end }} + {{- if $.Values.controller.reconciler.workers.bundledeployment }} + - name: BUNDLEDEPLOYMENT_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.bundledeployment }} + {{- end }} + image: '{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}' + name: fleet-cleanup + imagePullPolicy: "{{ $.Values.image.imagePullPolicy }}" + command: + - fleetcontroller + - cleanup + {{- if $.Values.debug }} + - --debug + - --debug-level + - {{ quote $.Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLEET_PROPAGATE_DEBUG_SETTINGS_TO_AGENTS + value: {{ quote $.Values.propagateDebugSettingsToAgents }} + {{- if $.Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- if $.Values.leaderElection.leaseDuration }} + - name: CATTLE_ELECTION_LEASE_DURATION + value: {{$.Values.leaderElection.leaseDuration}} + {{- end }} + {{- if $.Values.leaderElection.retryPeriod }} + - name: CATTLE_ELECTION_RETRY_PERIOD + value: {{$.Values.leaderElection.retryPeriod}} + {{- end }} + {{- if $.Values.leaderElection.renewDeadline }} + - name: CATTLE_ELECTION_RENEW_DEADLINE + value: {{$.Values.leaderElection.renewDeadline}} + {{- end }} + image: '{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}' + name: fleet-agentmanagement + imagePullPolicy: "{{ $.Values.image.imagePullPolicy }}" + command: + - fleetcontroller + - agentmanagement + {{- if not $.Values.bootstrap.enabled }} + - --disable-bootstrap + {{- end }} + {{- if $.Values.debug }} + - --debug + - --debug-level + - {{ quote $.Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if $.Values.cpuPprof }} + - name: pprof {{ toYaml $.Values.cpuPprof.volumeConfiguration | nindent 10 }} + {{- end }} + + serviceAccountName: fleet-controller + nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }} +{{- if $.Values.nodeSelector }} +{{ toYaml $.Values.nodeSelector | indent 8 }} +{{- end }} +{{- if $shard.nodeSelector -}} +{{- range $key, $value := $shard.nodeSelector }} +{{ $key | indent 8}}: {{ $value }} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }} +{{- if $.Values.tolerations }} +{{ toYaml $.Values.tolerations | indent 8 }} +{{- end }} + {{- if $.Values.priorityClassName }} + priorityClassName: "{{$.Values.priorityClassName}}" + {{- end }} + +{{- if not $.Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} +--- +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/deployment_gitjob.yaml b/charts/fleet/104.1.5+up0.10.9/templates/deployment_gitjob.yaml new file mode 100644 index 0000000000..5c3df9608a --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/deployment_gitjob.yaml @@ -0,0 +1,136 @@ +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} +{{- if $.Values.gitops.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" +spec: + selector: + matchLabels: + app: "gitjob" + template: + metadata: + labels: + app: "gitjob" + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- if empty $shard.id }} + fleet.cattle.io/shard-default: "true" + {{- end }} + spec: + serviceAccountName: gitjob + containers: + - image: "{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + name: gitjob + {{- if $.Values.metrics.enabled }} + ports: + - containerPort: 8081 + name: metrics + {{- end }} + args: + - fleetcontroller + - gitjob + - --gitjob-image + - "{{ template "system_default_registry" $ }}{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + {{- if $.Values.debug }} + - --debug + - --debug-level + - {{ quote $.Values.debugLevel }} + {{- end }} + {{- if $shard.id }} + - --shard-id + - {{ quote $shard.id }} + {{- end }} + {{- if $shard.nodeSelector }} + - --shard-node-selector + - {{ toJson $shard.nodeSelector | squote }} + {{- end }} + {{- if not $.Values.metrics.enabled }} + - --disable-metrics + {{- end }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $.Values.leaderElection.leaseDuration }} + - name: CATTLE_ELECTION_LEASE_DURATION + value: {{$.Values.leaderElection.leaseDuration}} + {{- end }} + {{- if $.Values.leaderElection.retryPeriod }} + - name: CATTLE_ELECTION_RETRY_PERIOD + value: {{$.Values.leaderElection.retryPeriod}} + {{- end }} + {{- if $.Values.leaderElection.renewDeadline }} + - name: CATTLE_ELECTION_RENEW_DEADLINE + value: {{$.Values.leaderElection.renewDeadline}} + {{- end }} + {{- if $.Values.proxy }} + - name: HTTP_PROXY + value: {{ $.Values.proxy }} + - name: HTTPS_PROXY + value: {{ $.Values.proxy }} + - name: NO_PROXY + value: {{ $.Values.noProxy }} + {{- end }} + {{- if $.Values.controller.reconciler.workers.gitrepo }} + - name: GITREPO_RECONCILER_WORKERS + value: {{ quote $.Values.controller.reconciler.workers.gitrepo }} + {{- end }} +{{- if $.Values.extraEnv }} +{{ toYaml $.Values.extraEnv | indent 12}} +{{- end }} + {{- if $.Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /tmp + name: tmp + nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }} +{{- if $.Values.nodeSelector }} +{{ toYaml $.Values.nodeSelector | indent 8 }} +{{- end }} +{{- if $shard.nodeSelector -}} +{{- range $key, $value := $shard.nodeSelector }} +{{ $key | indent 8}}: {{ $value }} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }} +{{- if $.Values.tolerations }} +{{ toYaml $.Values.tolerations | indent 8 }} +{{- end }} + {{- if $.Values.priorityClassName }} + priorityClassName: "{{$.Values.priorityClassName}}" + {{- end }} + +{{- if not $.Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} + volumes: + - name: tmp + emptyDir: {} +{{- end }} +--- +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_clusterregistrations.yaml b/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_clusterregistrations.yaml new file mode 100644 index 0000000000..d039fd40b3 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_clusterregistrations.yaml @@ -0,0 +1,41 @@ +{{- if .Values.migrations.clusterRegistrationCleanup }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: fleet-cleanup-clusterregistrations + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + labels: + app: fleet-job + spec: + serviceAccountName: fleet-controller + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + containers: + - name: cleanup + image: "{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + privileged: false + command: + - fleet + args: + - cleanup + - clusterregistration + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + backoffLimit: 1 +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_gitrepojobs.yaml b/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_gitrepojobs.yaml new file mode 100644 index 0000000000..c7233630d9 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/job_cleanup_gitrepojobs.yaml @@ -0,0 +1,44 @@ +{{- if and .Values.migrations.gitrepoJobsCleanup .Values.gitops.enabled }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: fleet-cleanup-gitrepo-jobs +spec: + schedule: "@daily" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + app: fleet-job + spec: + serviceAccountName: gitjob + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + containers: + - name: cleanup + image: "{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + privileged: false + command: + - fleet + args: + - cleanup + - gitjob + nodeSelector: {{ include "linux-node-selector" . | nindent 12 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 12 }} + backoffLimit: 1 +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/rbac.yaml b/charts/fleet/104.1.5+up0.10.9/templates/rbac.yaml new file mode 100644 index 0000000000..026c1c26d2 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/rbac.yaml @@ -0,0 +1,113 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller +rules: +- apiGroups: + - fleet.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - '*' +- apiGroups: + - "" + resources: + - 'events' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + namespace: {{.Release.Namespace}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fleet-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller-bootstrap +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller-bootstrap +subjects: +- kind: ServiceAccount + name: fleet-controller-bootstrap + namespace: {{.Release.Namespace}} +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/rbac_gitjob.yaml b/charts/fleet/104.1.5+up0.10.9/templates/rbac_gitjob.yaml new file mode 100644 index 0000000000..e2c93169de --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/rbac_gitjob.yaml @@ -0,0 +1,133 @@ +{{- if .Values.gitops.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitjob +rules: + - apiGroups: + - "batch" + resources: + - 'jobs' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'pods' + verbs: + - 'list' + - 'get' + - 'watch' + - apiGroups: + - "" + resources: + - 'secrets' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'configmaps' + verbs: + - '*' + - apiGroups: + - "fleet.cattle.io" + resources: + - "gitrepos" + - "gitrepos/status" + verbs: + - "*" + - apiGroups: + - "fleet.cattle.io" + resources: + - "gitreporestrictions" + verbs: + - list + - get + - watch + - apiGroups: + - "fleet.cattle.io" + resources: + - "bundles" + - "bundledeployments" + - "imagescans" + - "contents" + verbs: + - list + - delete + - get + - watch + - update + - apiGroups: + - "" + resources: + - 'events' + verbs: + - '*' + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - "create" + - apiGroups: + - "" + resources: + - namespaces + verbs: + - "create" + - "delete" + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - escalate + - create + - bind + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitjob-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob + namespace: {{ .Release.Namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitjob +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/service.yaml b/charts/fleet/104.1.5+up0.10.9/templates/service.yaml new file mode 100644 index 0000000000..49aefb490a --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/service.yaml @@ -0,0 +1,36 @@ +{{- if .Values.metrics.enabled }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} +apiVersion: v1 +kind: Service +metadata: + name: "monitoring-fleet-controller{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" + labels: + app: fleet-controller +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: metrics + selector: + app: fleet-controller + {{- if empty $shard.id }} + fleet.cattle.io/shard-default: "true" + {{- else }} + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- end }} +--- +{{- end }} +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/service_gitjob.yaml b/charts/fleet/104.1.5+up0.10.9/templates/service_gitjob.yaml new file mode 100644 index 0000000000..f33c9a5457 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/service_gitjob.yaml @@ -0,0 +1,51 @@ +{{- if .Values.gitops.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: gitjob +spec: + ports: + - name: http-80 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: "gitjob" +--- +{{- if .Values.metrics.enabled }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} +apiVersion: v1 +kind: Service +metadata: + name: "monitoring-gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" + labels: + app: gitjob +spec: + type: ClusterIP + ports: + - port: 8081 + targetPort: 8081 + protocol: TCP + name: metrics + selector: + app: gitjob + {{- if empty $shard.id }} + fleet.cattle.io/shard-default: "true" + {{- else }} + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount.yaml b/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ba27c748d7 --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller-bootstrap +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount_gitjob.yaml b/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount_gitjob.yaml new file mode 100644 index 0000000000..c769b47cfd --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/templates/serviceaccount_gitjob.yaml @@ -0,0 +1,6 @@ +{{- if .Values.gitops.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitjob +{{- end }} diff --git a/charts/fleet/104.1.5+up0.10.9/values.yaml b/charts/fleet/104.1.5+up0.10.9/values.yaml new file mode 100644 index 0000000000..37f39cc8ce --- /dev/null +++ b/charts/fleet/104.1.5+up0.10.9/values.yaml @@ -0,0 +1,121 @@ +image: + repository: rancher/fleet + tag: v0.10.9 + imagePullPolicy: IfNotPresent + +agentImage: + repository: rancher/fleet-agent + tag: v0.10.9 + imagePullPolicy: IfNotPresent + +# For cluster registration the public URL of the Kubernetes API server must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# For cluster registration the pem encoded value of the CA of the Kubernetes API server must be set here +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# A duration string for how often agents should report a heartbeat +agentCheckinInterval: "15m" + +# Whether you want to allow cluster upon registration to specify their labels. +ignoreClusterRegistrationLabels: false + +# Counts from gitrepo are out of sync with bundleDeployment state. +# Just retry in a number of seconds as there is no great way to trigger an event that doesn't cause a loop. +# If not set default is 15 seconds. +# clusterEnqueueDelay: 120s + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +bootstrap: + enabled: true + # The namespace that will be autocreated and the local cluster will be registered in + namespace: fleet-local + # The namespace where the fleet agent for the local cluster will be ran, if empty + # this will default to cattle-fleet-system + agentNamespace: "" + # A repo to add at install time that will deploy to the local cluster. This allows + # one to fully bootstrap fleet, its configuration and all its downstream clusters + # in one shot. + repo: "" + secret: "" + branch: master + paths: "" + +global: + cattle: + systemDefaultRegistry: "" + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +## PriorityClassName assigned to deployment. +priorityClassName: "" + +gitops: + enabled: true + +metrics: + enabled: true + +debug: false +debugLevel: 0 +propagateDebugSettingsToAgents: true + +## Optional CPU pprof configuration. Profiles are collected continuously and saved every period +## Any valid volume configuration can be provided, the example below uses hostPath +# cpuPprof: +# period: "60s" +# volumeConfiguration: +# hostPath: +# path: /tmp/pprof +# type: DirectoryOrCreate + +migrations: + clusterRegistrationCleanup: true + gitrepoJobsCleanup: true + +## Leader election configuration +leaderElection: + leaseDuration: 30s + retryPeriod: 10s + renewDeadline: 25s + +## Fleet controller configuration +controller: + reconciler: + # The number of workers that are allowed to each type of reconciler + workers: + gitrepo: "50" + bundle: "50" + bundledeployment: "50" + +# Extra environment variables passed to the fleet pods. +# extraEnv: +# - name: EXPERIMENTAL_OCI_STORAGE +# value: "true" + +# shards: +# - id: shard0 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-0 +# - id: shard1 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-1 +# - id: shard2 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-2 diff --git a/index.yaml b/index.yaml index bb692dc010..8366d95506 100755 --- a/index.yaml +++ b/index.yaml @@ -1072,6 +1072,28 @@ entries: urls: - assets/fleet/fleet-105.0.0+up0.11.0.tgz version: 105.0.0+up0.11.0 + - annotations: + catalog.cattle.io/auto-install: fleet-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/experimental: "true" + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.18.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: fleet + apiVersion: v2 + appVersion: 0.10.9 + created: "2025-02-25T21:43:38.833810293-03:00" + description: Fleet Manager - GitOps at Scale + digest: f60c237b73f5d1a595b9777b2a5191efa940f93778c98a15377e37b803a07c7b + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet + urls: + - assets/fleet/fleet-104.1.5+up0.10.9.tgz + version: 104.1.5+up0.10.9 - annotations: catalog.cattle.io/auto-install: fleet-crd=match catalog.cattle.io/certified: rancher diff --git a/release.yaml b/release.yaml index e15f1733ad..7a1c4f2b64 100644 --- a/release.yaml +++ b/release.yaml @@ -4,3 +4,5 @@ rancher-logging: - 104.2.0+up4.8.0 rancher-logging-crd: - 104.2.0+up4.8.0 +fleet: + - 104.1.5+up0.10.9 From 11d231ffb78ccc1fb3c5d4084b6af39bff5ec39d Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:44:46 -0300 Subject: [PATCH 06/19] fp: fleet-crd 104.1.5+up0.10.9 --- .../fleet-crd/fleet-crd-104.1.5+up0.10.9.tgz | Bin 0 -> 36274 bytes charts/fleet-crd/104.1.5+up0.10.9/Chart.yaml | 13 + charts/fleet-crd/104.1.5+up0.10.9/README.md | 5 + .../104.1.5+up0.10.9/templates/crds.yaml | 7076 +++++++++++++++++ charts/fleet-crd/104.1.5+up0.10.9/values.yaml | 1 + index.yaml | 17 + release.yaml | 2 + 7 files changed, 7114 insertions(+) create mode 100644 assets/fleet-crd/fleet-crd-104.1.5+up0.10.9.tgz create mode 100644 charts/fleet-crd/104.1.5+up0.10.9/Chart.yaml create mode 100644 charts/fleet-crd/104.1.5+up0.10.9/README.md create mode 100644 charts/fleet-crd/104.1.5+up0.10.9/templates/crds.yaml create mode 100644 charts/fleet-crd/104.1.5+up0.10.9/values.yaml diff --git a/assets/fleet-crd/fleet-crd-104.1.5+up0.10.9.tgz b/assets/fleet-crd/fleet-crd-104.1.5+up0.10.9.tgz new file mode 100644 index 0000000000000000000000000000000000000000..c3169dab52a51deac1924cab19d69755d48c0e88 GIT binary patch literal 36274 zcmbTdW0xku(ly++F>TwnZQHhO+qP}n=CtjrHEmAY@0@d==MTKORz*feL`7wNsFj&J zb`iutp#lA;{ZIkX7)dHKnn=pA%X)FK8MCM}nyRo_X{&ItE2yip%W2qJ8`+zBsVX_} zNt)T(0bTWddF-^Oar6s(Gz^xwCtRtsB%@YXeO}oUs5xWXeY(?9@O_#0z=w~{7>a?= z0!in)e+&d5(gXFwLkF^uY+6Mcpx8(vbpp}fT%OXi2(Qp0pP!Fm!?FSPc|S5hJ6irV z0Iv_k4<8TD<@d`Khk^eygF=Dd+Ze`te!k!LRZ9Su&->dZhQ&|t90Ffrzq)kT*g)>} zSKZCAKQTPZIJ$Z&%iAZiFtj@7QH@t{+Vo`Z-{nv!v5eHx`J_QwE!4%2jqGwNqxr=o z6f&+0ztDr+Nf3%-F6MyTtXT>U<)q#hbozpl*yg_?w+-OPTx!nhd~VJXTGO|tx68q% zU^2|hBcAcqem_tck3Phtf3CQ?zpjw!qb~|99x5yz8t}_R$%B8vd_TAOBliu%HAD(x zl&SItNRTp!3DZv{GsBGfilTl!YF7#K3-HN39wfWS2=U?4k}3UVJ))skFk*G_>hWH| z+pWX-z1;FI0f?K!L!c-zK>p?Z-I}$9LsM3k8F>#s zSHI^6GUt9oS~qRix`Ca9rXa;KzcX?HpXwJq7tiM6SxKm;0{>J!Y}*Lv%zCv^V7OMc6lEJl3O}7>k3M`R5jC_ z^OdzsJy|#kLgMvU(0X^ih3p;;OHoqmin-^xN}^-(_Y9ssQFi@?v$C2$4?Gfn0J;M& zw?F=m@%yeWkH_;5xsp*Drk6SEckMqRCsW%MoHT2%RbM5tYorrwd|pBuZ`(Du0s}Xn zfr>IaJU8~xelqFFrH*PaE(-MQ>a?S@V$QR_X~kk|#0#M4BE9I!f=Q{jX9(K-zmKvR zL$4uL0FE)yXcCxYT2$!BP%NTZs+nY9_hy%2U=~_7bV$S34z3l<5~n%=BEe|El6`GS z+Nh-&u>56fnwu3mdCqI!P~z{ZOJs7@LI9;UaaF082(2%$JJffM5xSGJuWU@|_8#5y zl$CeXw8$3QkUP9zjJ^3@nB-)$lTRj5gqlfcbX|d6n+L5vYXXCiPaMv4Co1jP1RVKP zU<|FwqQ?)-(9=XMtPB2_1aql#O9!IHQ7tRv;k`iKR~(UZWQW)Cy9(OU|7F;R>s-;{+KvH!m<+Tt7^>_6UU@B^9Tz z3mUt}o?#n02_u`y8q~_r)o%8rJo7VRt7oZniCmSa>5r|X?WNqQ&#lA7n#-d}tI>|p zFkB&)P0Ts|oHJC*IqtkUGZYBlWZ%&j^iw_V4L-ZEba|&e7O>(0`a1m$u7~TQMy9jH zY;d{e->sIE+ewFeyU}k_dfkqf{L~uY1jB(C;5Yh^v*p z7e4a3j4goANx27+dArOL7$FjOymNUd#p*p?#bCEYA$(Zsg-5cOM;^?>)pgjCd_7^ETHANLqc3gWD zB1WYyq#nq$j;b_D~AW@}HPe@k1?(J_e;AcP*~%8Kf_-<``Cat7$)^A;JQ+Hx@F+xjS?dc=t7^W-~`wjiq>Vbh!J%6D83ZU-x zVA~i(g?$!H2~|XXhfvT?7Klq@|6rB0{7L-HZ+Vi69WHaDEg8K1kHS^xb>yQ2F&d zvP6?Z;&ySFr(V6fsA@SH#| zR{;h=Bt;YvE!dtV3agkmVu?*V&B%O>S&_2L zl&+@qs1hogAPX_eFOOBGIdjAC0uKSSRfaTsvQA67(f*@m6A-KEPvM9^aptI{la9j5 zsp*2&iY1N2G})u@R20TyibYz!EB4WcedTdmo;OaJexu6-|2~yqC~>qT7C(&HtB>eY z)~KFv2(p*%Syq}zPikizuRlH4kRziTS#XY317Ogj3Ss3;Y|qtE`fu#0R<6^4*SjR150f-8Z=1p^ z>G9r`Z5b}Qp&ibJ+l4^aJZ5d{2N%rZ2i5rQhfAwvD)Pz7P{XS2O~i^@XT1(I67fF! zzW1T}j*h%9`O9ma0A1ZK3!{x<(ldLxOr^IAC489;wX~S>?MhLUT#WZrt2d4!$wc%t z>Bnoq0c=}1WaJQz`62mL)B1?ZZcZ!BnqBP2ctHJcdhx=!Kgac~x9ZK85 zE$nxBaauI9K_McOwKcMlEw@H)>eQS8q4N>@q}3~YVvL6i59q475(k=^&qoY2dL89mO#z6&f3 z^BSsnsFogAP4gK~=MZK8iEx3f10V4p*Y81ZToDb=)F>*-*)md+Xx-i(*CMzO+6osd zcY_u_DcAcO<+(+ICd1UaZ^Rw(5m`k7)m$liK)6WvK^ZKCsM+((_osC#@tBwZlc#agoeZ&aOU??2f80;Ufuah>P)jS=)=Q4gbOfpf+ zJf>iTZW2{PbfHK~okdESMfu}6CYmR38rzO`SF;j~LC1!lftCjTD7_<(s;4crMg?5N*{nlQBgLhvIT>fB~ctglz=r2$45t(h{)Yt)Y6s$d#6C((q(S1fF6G&fAcOD@IVCbLTBgq0t z=3KJAjA|c!)X7>-%XE_KRy^$e9$;MR^T=iw9P_=)3(ecnk2!7>t9L8PHpM3o2$Q1Zme(gH5%RI za2PBKhBUYz9>Glkm-;iv+)d(eDfNcWSP}vkQs0rka4dRv0PmOOa87=qWI4Bl<(`k9 zJhvpdW}ouJULrBVw+=r=5^6}qM$lwS;_L+0?hfEyG@inyy?iIlALW-a#2JAVSTWyZ zt_sTLT^kZkCL3Xjxc555U~27Z3MV5#wDtM=+m1H^9@51A6WLyaAM=qXp&1w1q`77H z9oHXgHui{@bB|+g5mg{*!Yt;usn$NL-=93Vja%lsfE>vx5E&YvVYc9ULYUrtWo5r3 z>YJS|qVg#FNpW^pLIKn`c@X#J~7nmLV5zG}AD``D1ihX;*$um^ePz(+|M;W{U^VTX z-AMd{PTxa|3l`XzTke*|X32;brPPEq}6< zG`_?cfrULSVdJ@KH=4RKpsHUk3Cobah5=r4#YW5;{!KU<_1Mif8Yqt04j;g^;?%NZ z&24?nj>o3(k%ik9;qj=7DR|$k#)C9T85^x({S;EQsBQP29rJS>gRDJ6aG8OmCp=D6 z8r+KBDtxDtoxH-}8MFwM+L26^ZAG`!FgmEQ{+eD0PDFT)js zintp8zC-am5FU$-{B@>pRpaQU;jIj7eENq+WU>>$-SX~77$YDp(X}4hrmzsPn5Yk0n(@g3U0NY(a%ASQ@fc z6wo^4f|lC8RqB^87R|~cn?4BwwGj(ObspWu93Fn%v{T9n$FC?sd0~@JKlE}7bO40H zR7YlHth}NxUSOV(%B5DCS(7SC+p5$0~eoIzhLs`WWanld0B1VHpERTo9t zxY1}hQd9jR7!OLj8XM!82psM%=*ZpBlKO=dHvF}@i|rA)32a+Dg-Cq>s! zdUS@P9eY}%+1^Jh-}VtJWi0SSg}!w3-Sy>#*)96P=0MePUL0U2G)_dU=6dQ^%K{=K z2a1FSD(0&ACpzSSkWhg}!GM^61!e!I2L1CZKvNol1ZDye7XH70#qEFQT?p3y#fk3z z8`|dp5rVMf|1YC){t?pn;4*0`JPx zsV_YZQoKVY)$)>7JOP{!buC0)(Byxtlmc&FIom!>Tcyu=?*;K<37)#aArgLXD4` z96q1u?_tuY^^xARJtJE&tzQ-HhPvzK16(CFI7?A`apn>ftV8$$#x86Lke_h*u38Vg=3RDWUJJmG_Xz&D9l zIA9k~8XfD~IQ8Dn3;z1_^B?Y+rpNm6zO(hNuWR>wuU=JSnG#D3hW#TLdewvbtTOeE;Bfx6atqC=F*ANs0Ocq*^lb9c;{UF2T(%BGs$kWWIu7b3NkYQuvzr+-T8WnlG~S%cq}3GYYetZN_-EMYTA2 zaUdP2&{DWdo_79j{<-J2p8gMR5g-LDBe{^LA&&ab)vIO^M}xsPwBYpqHEIdYr4Umn zOSF(3hiiEL7fB^cpD*DJcV?7K{uOANK1iCKWtAwl;DBajs+g z%pKC-GAn_wZKV3>`yKOL7^8s`3oWn3*d+lW3s| zb^qauBeMN|^X!h0Dw2uKUOFcZq-_tZ;*Qj9-`0oWjtwz_G%Yjv zsrV+FZTxJ)+NG9#Ob$^C)ON!fvEOy?xvnow>BFVL%hZ^2v6ww}wiC6d!SnYJ(8UtL z(rd4wzpXdC5705dl-iH$;OIS|Q1eg8N~~RAPNV3h$HI8!rs$^R$@FF$CD&AwL}P0I z?T1)0@mZ=$yxwD?kzM~p3B}tykFfnigxcHM=7h7)&`GbC*F+ElK5V2$A$q?OVT!du zR@yO*KADY?K(aSB;xguAuopM20 z0N?w92C*7?Zd6x&Xuc>a9`-^*ZZVj3f>CpmbvockA0K60Eoe_F@LM-TsrC-(;~X=Q zH3qGoe56}k$ZJ#?n7fIy(3U#cup6ur%VUL6u<@NT^7e-;*VgaP5xb0g93yK5C=kr}6PF-$&f4j?RVcv>8QOSNxpj0j%fvEKJ< zwKw!w$$vt{<;$G(US>)8E7===U5~7zq3nN8Iu#`V08e2+AAbod2URDzxFMT-33rR% zdS{btU3UjSpDHRvz2EGePGDdeMVVa+xUkBdXbK8@*fLBV#cHsjBsff6T$yuq7kTF; zG_EFf{dFRmx~aH+v!6lv;|RJv?Yxedd5(wGyFIJM#~jeDT2P#qwrHwa?Hsl}tsW=a z?)?LD1#)$Pro*soj?^D2k+i!j;0D2oCPsjJQPYJHqM|mZD))e!b%Clg%4yDzy57I{ z-jBd3mykv;gS*@FyNu63PDy(Lo2d}%lz^^AWEqrtw;e|5LuZHdkSD<4^4S{sS!q4IA5D~us!aQ;TJy?Lq5!vcXwk*sKkxRIq)Y5=2 z^u@3S5xP`#D-6a}4kT54qc}~#Z|iw0k*ac4&F(wJ?7-*-}ie^K0TC3Hs zV)M|6PTLf9P;m(q5zW@ZSa;a#QpU)0@YQJ&N;MJB5T*7Cwk|~i&Ctwlj6^kcvYD-( zQ$wh0_9~IZw|ktwT+E+|67X;-M%B-rwUBXwyQJ0)`5ET( zhAuATdqtlpsS5IE=Ea4H+$it{Hw48E!t!=;QbRB1^6h(56np-t9OqY!;xk=_Q8fe% zynAzc?&;K_a&V(6 z-(?(%-K-G5r9ude;YwIeHYCM4RT^#>w_RFu>e}sL>auWL`x}y7AtQ!pfxvzY3obY= z?}gA_;EuCF!Umli;XcqAMc^}IA!fuK_v6eX6d)7cXiyc*Bb~?R=&k&TB>nMjF*1ys z3okfV8;59-SHc`qtn;|qZA`}}Ny{fZ-!$YtZb8!XC+#&mw0Dn<6fVX-&jRxmG`d&+ zT8|cc$QKv(an|s}gJ`k(xR+pYV@N1UVvEi|f$1qz%6ELNeOXGqDtJoha!8UxdTa| zV)WLTOOc}}Je0U9sC?mLOdRGp=Uk8mwDNQZh{GgIbP8?Ohb^a1Xci?OLo>01JdtU- z+;!1=%=X8;2&a!oXx%N01VW>Sxl1FBTn9=!#Ddom7(Tu|{_6^-4i4Vq1$bimjLGX= zhsD-&pf*+CoSs?KdA$V^MZ?Wa@?})|A58WJT5=;GJdxI0E^^M z31O;~wIC_{wAr(kDB+_(0-eBH&9FKdIF>o@KRh4DCD);jeRXyp3JP_H3dZ=zWlit* z=N)z%0&ym~>$7-FmmI4yH*15A#s{)%xa){Phg+U4icemsy^Fo{@`_Y4k*S=_Bsh^d zHQSgna4+fB_?R_AX*P(|#Tkof>*D8j-bLc`f9mx0dgd?xK-^~MtHqY>`&)o3-=A3t zN$s3aW3_#g%#&-M^4|5K-R~l6wuk`!`h)8_t_#Ix*$Fs*ITLQxzc*vF zk4H#{46b71Qi`gMrKUCy`MwLEYbX>I2aKHf5_E?z9u~XCx_F{V89NP{vpg_sn|qq= zSTTA~>#d>yhz8-(#-#UMUxEz;2P%sQ41lWI#G&#zOc*x`(F^J#l|>70_xeP2TZ3gG zBKiltm(OPzZLt__KK_A+?AXZPTHx+m-bwgm-;9wi_q@(}w* zCzi(3bo=}9O9Qz;DAfJJCuMs)INy~NE+kqvI`@NjNf>v~hT5bxo0-PI#AiDm=!dn& zZ%LCmRtiC~R%Lcg`i~h0?vH<-xIa*(U5?o`PqI6yNr zv%}}-uCf&?DZ$gXnk-e;FIcTG3=?uidZk-85-ML2>#KQnSJ$x^JrD8&1AVjX103@C z|3E-m;B5#~&nOsrvek~PLweOkAKw39fTrI@;^Ac4_wU{u6h@r%e;Cyec;*NrXOKEf z)w8#_SxjnQd`D(3Y_i2rUOwL6+&xq3?kAGWIp^k2j;MiYRN>k$zqOsjfQEd~r;6TS z^G8I@mTREQIa89r8q+f2UG$|;gu)fc# zAyXn*Brt+YRoj$l*8Ujh-+j7fr4b}Qss?zSgUYwy&`I|-4wGIQW%uPmud_PoY?_rSs$+#B@z zKEG*<#IJE~7ZZ)q==#dA-%2*TI0vqcfAsuA|4cY0+F9;`L@@jCYz8;bP%l`9=g=Hf z;-c+vtjuwYOksQI7ZhKuM+mG$&`Q6{ntvoTL7RRfhoR%DI^&G=X$XtH^53{1B|#{R zpdvV_(t23or<@adZo4Ak8zV))%yPvF9wyjX%8*G{) z38*y?bcH?`K+0HECxaPsu##RfVgjE*Dl|@&U6j2H9x>+fwTp9r)F;c%Gz(A2tG3gOcSlXlp^N=U0Tz6m3*Yf~&DJfmM_MBYzwhSC*0v8P!e8K9f4 zgcpl5w5Eew>$@*U3oW7XCjDR^_f_oOTNk*pdnQF2nK#i0wU>o8bNI6Nm>H=Es;;pS7d#z&v`dGwmr$ zRRV=7hKt7Uu*b5Wro46RgDA-u$5}?!jKy+dJ*d2dZu3X%#YGJPI_%4|x+EXSz!Z8< z1(Fm*Q}k(3_*g`ZrurcER>09aR7SSNXELj9+;+iVxN z6X?hI2i}-P?9#0fJa56+M9IOu!oU)dpVfEg@;M^Nnw+Yoz)++y9Z{|k{v@%*Al({!Tltes_v{9G&ZELfmLn zQf-eeXI*a7;1dtv4tQ!7F4}6F%*9hEwR-XMftXF>aVkz*vjj_8%7F9jthFCT$@h-& zb6=33IZDJAEz(aJ$~ghSqiZA`@Or*n1@ID;AgVdpGaB{FIrKE-R19UnC>3eFiJS(& zpr3)wGuQQI*G}m#YehV?8aNszr2k|RuqDy04MS@`Zi@ib%FcTRo(QZn6i z2SQf8-_L0uYc4A_2ee6v?LIg1NBs3SST&Vw6eS`D^Qh36xskJ&eJcj@T z$E;z8P_NE|rE!TofyiWA8YwyE*1}&EaY$~k{$}CL`E6gq+Mf+m*=%W-hmgQGf|s*)9;vz75@E1J+!?78$g~EOqh2{5 zc(3ij3fT%%CyJH0W3<%GVqKFY;=jh2RqJa2I(RW)8!KOc&DPpg{VO`mvzQsK=5O|u zA^={%G6NfRLg`8Kpv9O}Du_*`28?{-h*y}x6A!mjI!=Q3(bIG?^!yYNsP5XUKBrg_ zj{)dU{h?YOF8sF)^`+Tgcf9ijn|QZM&IF5@ZAsN@F3#T(4X%(N2LTyedoc|@xRvPL z5{5LcjE*#{5T<6NlWL*~m3qMxzs{(uPdOQBTL}n%VP{209R@Wm@CBO`Edps-@{Z1N z7Sg)T8){h+4AQbfu+nr-EbePtW{5>XZVJCZwuT{qw}*)X+lzU0=`~wQEYRJ)wg+|; z_w3MHu;Tl|UIPt=jR!dv>9L<_GSas)5Mp41Plpu;buC~z(zB9lVaJg!=^sZyTnh7m zxED3`n6f%*%+&4Due4HlVssM#F|;y7Vqha4qc5AZ3@+ao>Ra-P>y3sS1@+0cyhkxK zEws85s(q<*6u@avE7aA1x8;2-A0t$cs$FdN$=WVWk>~vr#apE*a@?DxDc4N>QWOt; z%;%=ahzkEP$Z==g4(YiycM>8mgV;MQ8|NKeS4fz|NI)cj@dJ)XJG@mbV^*2OW{bI> z4B95GK_i+(sVw*-LvFgqg3V6Y#m2|{Tkd6kanE0k zaLbqY z#ookDe+LNtM)cd5(O_4>pDCcv_L5h`TU4b3?DqGK^{jXp=lVbX3!W(D%&v4~ozZ6q0TB$@XdQttMr;z}v zblgKT?uQBFY5Hxuddql!#pNE>PM|@g_=z61yF{`c^As=u9270C84c&av`MV&Vb1fC^!jqBqzHlG( zw5eo0cPdT`Lm9As6;-^=MwU8x$0;SftmwGNzVh_}U*DNmTV3F)Xk)?s43_0#7XmrD zccC1$>n=Guwj6b9fNTm3U)4g0opMp%zl#FhTe--O?-R^7CQo`Jj@7nc+E@)`{y0Z- zr@xIw8Aa;lIeoX_<)QdXTd?XLU$8GFmRuVq_!An6P=tC$nN~R!HO!%4?!Z zp!-H;hu6>L;KR9vg>SnW;#Mjnv<}D(@S<718UT>UCNQ;E@ zJ0jKR>~obGaEkeP)(;*A#|N=2YI)fw5X%KxMi9r2oE6DXi)~N}j>?TEdq};ll+eNfh|f|PVb5Ep4RWdgS*v*j7}Hy6ow9Qg>~em9 zhSC?IYBy2oEB5jg4mCOk@VR%5rs+UNG%wke2vDxuX;HS;<5fM42lP%ro z;H4woQQgv4-AlHh?+Ie`rClLMu?2aavpxhrzy>53bU>){JGo}<0F(KsdOC<==0+bt z$eh0;7}Ss$R+eOJ&kAmFQ9bb&sk|!HR5U5ZBV%u?{AP2!TF<`E@+0}|aJMYCM8J56 zQs;{R)RRdTP6dHEWv3qz(rZK}NLHWI8>7A+t0heC)`a8K?4!<}N`vRvoJnI|^%9=1 zxb!j-YpD!Uu#AtcTNfdf7h-5K7z(UiPnICi> zmUdDn1(gW`y!FZ0%)h?+z-kjlohH=Z{8CI~^&y!}${vVwIFXf!Jyc}7HT$SRS=f7m zK%EScIm~ICQg`9@3mgDx;s5SZ*fIZT~5PwBLH{^S_S5@KPS=|9^(w&F-;c3OyuD!`}!dvNZ zNQB3QwP~F(%Q|z>FK*Ae{5sH}x??il#ka;IgD>pK9aJYM-;THYW@Pr4tOJkyi;srM zVqNt;p|5ibZrF@>!;9aeq<9JF`vAU)$U#FFGZ#ctweFXAz?1v9S{~FvoAenAOQijJ zZma@aKHmCBY{BfS+=RyF<3v5 zmi=r_`JwaniplRvoHiI(z{&FA2sBFR{wz{p#9~QYmLFUqMQ`Ct{y}$3);S0rOE2>{ zLZR$Jij@<24Ra4o5NI3Kzmk#QEwT!C{{ll`T&Z;7ux*ICZ#%Ixa!|j|%aQ}^%Ju9R zI*O@S846D8;L_SBVwZpSc?j>)$cnaw>GL17gGcThZtp@xq6xr5g#z4&%4q6@r_#I! z6pxz^6HY$r7@iJ>atXx{zT9N{A0H^KgnyqF@P8+MCiT|eH$C+nQD{>!vh!_>1EFRR z)3oy+k@U1ZjM1Pov4vDZtn)J9igU!a>Fn^lFwlmr7b7v7(NOvj`Z0N$@dB|PoPE{$ z!dSCaW?Nj}f)C6NsjwKw>A^HSj3QAEQq^7^50&R56$-91xFd>t?aN7NdGUbN2CTh6 zx`8TjE7<$+1nfz9bb@i``1*9BU-e`4)Pd-s>EjZ3GzM5)6NLxlyatND__2`>fKzw> z;>^l?3$F4g-*Ndqt*;6U<(>}kI)5IJ8PT-)$M?T}Dz3{0J*`gVm;8JK&wsnKe_YHb z1Y~ahe58FWrvUiA8@gUzCBVg*Wjp(|*E{SD1o+r}?Y<}7#W!PknkdkL zBj%=$7@WKhcgg=_uOm5QBCT{xI_*BZuj!E=1Of5ckdnklnUx(w7E=qSC$+JU<$=Ln$8?YU}4Ed)vvZmc1(}&Y5{cUyY}dK zR5bMQ$f8#=WoJQ_t8YUijy#zRD{RC4_@KV28d{CXbOtY=i@IUxN>|@r@1MIG0kebX zus=2+ZU2`Ebd`yU3A2Om*EsR(D$vl}NyeP4$LKSDHa3Rgm5=4L!HV4)tG@wYZ2?Jp zpb$-B=!ZOkE#^2zspHUEbusvy9~~O(0|#Xp7#w^mNWBSn$zq6_OLtB^Ar|Sch#yEA zTp7}no4_E~4|gZT!)@JspKdc?gn(SpGePs|S^={DfPVMt%ISyo|1P z>CB|y>EAswjVR?^Js{w+nE3x25}2bB==1Zg&d~S|CkQ;M?*BMJH6r9jnNtJ9V;!as z@nA6k+qxj=67VZOxGYDfB;ytUj zctbn_2bp32Xzxwupa6Ej`HjY|k*%y>-jfpp(-X&>2v*L_U%Nj`vCyD*vb1HxWw)De zuN&)fr~xR{r$K#xiU`S+u_rrjHxMcX!7FAsZD|H({!xU0J(IF;f=m{xF9pjkL6WQJ zo`S`D@FFAlmCqND6S?Kr+K~Cec{}LUX_o_L*#$3GR%D-M1Yv=(4C*L#!gLMvIx?Py zI&jI!St0SVit_UE@WPyd>0B`N6QN)vB1imyy5k1-$eKHm*sY|byN&BcMGB(wA!UBp zA7LQ45;E1EW@)l=-cc=FOwd72D)o7gAfS(@)z_O+BT?^=p+R z>I9s7nDA33R$e{8Y5o98J1J#vQo?lVHCmOP(1z6zLaT91G47reoqs@4Hq*9{e~F5> zBlS>&gd;3XklS@mIwK;Fr)obV6Y7R~MVxzUr~smxY#$U(8}zvkPDNdsl&yiq_j$ks zs@Rgi*nwz^Jc>1gsJ0XK#~h+2@5J-J^*mk?Pc5zOX(V)Gl0e*co1}f8@3+Js6@^oC zCNQNg;`}P=h=7e~{!TT{FGtEL`n**VpDH0Wo>@n+_lkO`)^8v9LOk=4So%}{7Ly*c z5|iyq6Yh>rKW1`8!XEl;#kTKn`q}=FYC*qxj~GB~$}UO>NyPL%+d&d$kiur$8NdX~ zhvFVe-9NXUWD35dxMqR>yT>O@0q4}z_nI-}c8OD!;Br0Gq8Pv#m!SiWK{Z+=+-t>A zfBZVPZy$Lg(c?_K&o<($6bN}e?$^uQQ*aB$IoEfWy{Jn*Xke@wikW|SN{-LY^qTVT z-y$877d-d@y_OV3M#H(!l0n-vcGenUVjWu3KQi>BPRwu3~v7IX{^R{BNX zpzN?GVA--|RE9PUousA3bA6}bt{`S{uPA*HzC8_2 znD#=bJV#>GiJ7N0Q)+YgzPOnof}r!<)21?1~0ykplc$ zuVMLAyMxj0xJ^uC79PFBh0xrlyf7G{XB2(0jYUn}r=?=?bAp>gjMfKT>^ykRab#a2 zI=SR44XFK}gPAwNJmK+-$bD12 zm(32wCUpKDgOXhZBgh-ux1qtd6FgZ}^f_eteAv5d+q_Nb@1|yTLs?2@6>49BC%AjX zj8w9@uNhm>+7Toutx-vYQ{on7y;^=NGrj3G zd^sJUG!gs~@j>702&FMM4vVlWJL&iR=(R4wn!C#T3Z6ZO+H zkuJX1EmfzOp!G-Wvm^T^mmH{*wrF2tsDAQb%;=8~R%})HeQfLzg391lHz{Bw5b~n1 zA$XZk%C>NwWLoFep1to`tlZjN0wVB^-T2M1w7C?N@y+qD(*EDb4|@bmK)UbWJcU& z)i4@DL8^IS!yjGM4Ke{sBU+JWn*zOx&38NAG?((2FD&D_a^R{dUp5JehOXc*$I__P zVt-ME7g#uKSGkosIzR0OQNWzI$E6&rdCTr@9q8MAl`+So+hC7k2fLGYgU4fyB6w(b z5Q2J9rvRAeg_FfG{06^Y4;XNr?j^M+TVkV`vAufI>I%}n045}6P;EK(KL1c!1A@+j zEcV;VAidy1jD44h2aewI{<{6X2a5}KD-**Vm1Ti5t=^vGRm~%`WOCP+ zshzdfW{)#2B)$l(MGq#o~p5gsYYCgKBm8T;8N!P&6|#iu{7DVF)Q*U`-0MIl1#y?^x9+k%tgsP?eQE@-c+L;5-nd-?0& zJasw3CAg8*9kX@HW1ZI@MX*&JGvKS`!+{8^ zi)1exG(IG*(T#@}DcpYG)BPYz=ZLDUe`TtP_qLE_3ejIKtRM6EJ54=Y?QCXjY>#EX zuo*(JddzqyrA2jR|C~7%?4Xlo=<*VS8rkJDn zi#tfRV}YbruVM8OI9DHurg7HbE2v2BM(S&MhQL_R!>IfY_c);-w z{~f)R^pb|*7imO52z6T&!2@wf(QkSBmB2p;A5suyl=0U)PT=E^W8I7a*X#B^V;zv2 zqoFt)M%WAlCkZLO&Smpk%K*w`M3H{VD zCBgVdBY2?nfq25WAxxXHn#U}1#i^!g-#WUy_<%xUvMqh$XSp4BT(?JJPJl$ zBl*Xi!jxWM4|qP^*vJhXOe^&%;BHBFHHrp%zZz<^S2pN2ZYsQke6eHw4=3{2i5cx} zC3h+fJMV$jNI>_+!~3t1l-b#L>{JW4U!n5#g15)(hbtM2CKcP2glx|)n60fNU=3p6 zbhi;CVlMnD65BQm+Xy2)T+(^MtG-yG?U?SqK1vG1LFe2N*?4l>JQD7j8;961mE8%3 zOnW*p#UT>Gg-3@&p+m#2agj~{7$w3(<6uEy6j$({TTI873lWC4nHrvJ$VGZDs4M6= zb8Rw&8L+F^<>r*3G|Zkfq6Q2&h@sYQp(qBaYngHD#eW5xey@G){a9Ij&Y6rPEy-9dWA)yrF8zSJapkWxdn^FXIy{#T>$P*`h!zD+ z^UM6JT07iEuK<)z)bD4sXwCY$ydc>MG{;*$SOWd~mpMvd@ePq|;tV+UI`aZ1vI#Tb z^dEZbJ64-kMg?!e|4$-MnD6v*w$b>uNOx=&S0#HHdm(w-5dT1ed?Hb&h>@Z6EpGQ` zi|6wb+r_VivKj|K)p%G$NG>}AZCB6L^4Q96ZmU0!=~pPP#nmCxv>=R?hs`~%PT<@IO!$?NC)Mr?%U^>Egy($vo9 zrNav3;w|6f{+VJXlEwENR-%EFqlRJ%dSGTp*XQTWYElCJsK z++8+~=tA=Wk3c71ZeWcD;eB2Fzq<0u!0x7B;_xj`zrE?>&Q<#PHhKlNT|B-gak+7f zTpQzw7G2kVp#R4chbEQE*hyI#)n9BbssGiM+mm*XqoCb0le7|Zr8u(;5^5-kAXeYy z?g!53drs9oa({^oD|VtS=AZvFv${7`4S8+kvCe@|7)sT`Ou(!h%D&wqlvgi)J3uVPy*csmroi)Wl{Gu zg(8H;PPPV`>VqPA?ZNT%cRWM4@DX`I$e4G0M1_LmaNP*Uq`WEHxVw!SBIh0zHG4EE_M@6VP( zcupp8&Ex08p>1!ZoV?I(ru-bQ_7WrqD(pfZ(THZ~EW}Dy>I|@`meQxfmg@@w2xh%M zFb=ledg-)NUXb<890|C78mASpJ}3SI*u(?>f}=ryiqc<E;n^0le2sRLxr@fP%kmx=Aef^tcRt>z+aL(XFSo*O>4e(B-;P}e zk$0k1S|9XmqWahM*g=91Zax(KJ>TGz^eB$8Gc;ZAyQ5h*>PZSkAA2}ed=exJQa({k zK1G5Abief1-ut7}A^X5zu4Toy$>XftqwNHVK@;FcknBO z3ScT1_-s)1nDFz68_iiby%1X5@aHH$7=IvNDKH>j`&5;w-f9bt=|cg2s84jzCuuBr zkf71_eXDJzGntT=GV|snqUpwspq8BAC>tDh7C&~TUZD|Au*zPwC2_Y+m%}h9rfT3c zLd&%73$Q4#d`fH2;}H?UEPK)^RtRJ@r{HHu7L;Mk#<3$WBfRa>fQw^zE=t~Ghg)$` z8c}!R5{0(H5yKRwote@<2FIK@sPOtCZnzKs{z=%K8lNbO3e#!sO|vh!@@DI<88K(d zN|`rGy30B5!$()wXQ>pQ^3~zpcxKW&lm5wp6fzr_%lLoc)iZ?vzBD>#wJ+J99Goyl zgVWyzQs_)@!9#*5!`*>w7QM6g!S7~Q0C}eE1cUc$uV)$=?E^ti_+;m{P#0Z8Q=y z%4y+KKK#fqo;hSu)Ibmc1>$&~-HtxR*FoI(RS{sk8R*4ga zGYV+fOK*%bzqfRvA+pu%@nVjaEr3_y_^eh>7p%UE3_x3UAxyUoPYmn#;k@@k%NAvJ zaS1#}qvgMdzxnd$SKM5moEvoK5P0ZX&Kp&P|I2)mZ&MK)7m1SDtmz5s20JO%`?8w- zggX}CvEni4Cc`I6;e;d5mdUT33&zhT?lS&5r%M^rjR(Gz)f?0FJ7x=RXA%ck9vX8w6t(8?#jsLeRC?}wDZ9>)S+Bg(8I(hq*!HP3;a`zt)3Tm&p^d@Q;u8fK2v6u- ziEs#Sd?JJadj53h_8UPf7oJmx~ zo8J=#xG%_8b)Vb$JGT~H4%8dCv3l;SFFG?8&*X$Rf8~!@hvOQ*KsV(?Vg?)C#w|Eo#Tk?*g2l80rSx1{TC6UA$xABRfsU`KMlI-NuCXBr)Mw69S@d z0HCj)VJs;jA4tMsK}>X{n3V1Ctx76UDOzn_C5xh04+HixiMqSIvn`#R2<>-x5Y$)Px z=`clLoNM~E)BHYY+u%)}opG1^dfF_~I0=;})vFn&$W+rb1<+Zt)S4QnP0z>+F#wMg zq|+vKiTDY)x_nZr8m;E%dlx`l0#yv=jLhY;n!X)Q50&zU3evfb_jvlnP#Oswfc*-q zs!R+1=X#my0*prjctPWWmbGknYH~x>!WL#$XMot{)r*76mOEbK>VO0&|Jv4Ux7GlkMPcGw62PP9 zz(b+(Q1RiZj|d9FKi%}eG%6uyXhNDCfI~78eoEoUx9X!k>793I$<)qoNJ z{(^~%OX4$m02j3L_n-GWl=!WZxT7S`vg)&6{V`@s$ksa65*7(=LhyWhXnH`W9k%~S ztdj6yEc&)A40T4kNJ$3FbW}8{^4ehFj9~q9+*5#na=%O{dfwgsICjpU2orY~f0-=_ zMy;q>Z;@Qa;ZxdY|I1xFhiP%}Ffm^9%KqkMcLz2k!W-m(;(f3s!=EwCOGFzx%}$_# zq1Jjx@p@3>E~Od-l1!zzV`mpF>wGwVX5ILyAl4P@qQumW5EHP`OCg-I7TGThy_Aes z-U~}+2T0nn5b_)4tkSI!)B1w+4wkDGU`$^19#i4gv6K!-)JBU|9K|(5=&dF#Hd?=0 zOST-2=vtpeTZI(+&jz8^X367xK60hCJ(F^8Su?{AF zTvw=So@UNz)BRWK;be~4)%W~G9oDgTaFTI1C7c%C=9>UmMS;<o zL|m_pS#5W0mL7&?l+NjYvSOKzKjzZgI2+XoZaUihU3&e@H1r)n2RHr9)=3Q@rc{Bs zGr74rA$AaG$>JIBpN-;_AMhyJQCoeDp+4NNrRsCAM%^uq|xvN5NLh zL^hnJz%@E2P1X9wj)n2Pq8yg(8s@gZ)lj*a8zW%h?0L?S?7t6Qek+wdn#gQBM{Tga zoQQ@pf%YLWy+zfZ``C!$dbHeJW{1i!T;MtBpm7y{2JLs}}CH_RqK9uSbUcjGu zWL3kWUO`((C6eOrB(2PHOA7qgFt8?V{EnM}nC144KjzrC2V7p_$3dQ*3qGM7>8fkH z3jb#8w%;ZKGEpf6tSwGWQqPv@zvr*&ML9(g;%7YYPBv0P+RGL+ww5vvV+e;BYES99 zPe5m1Ef=-ffdfPz1~~pPBpwMSrblgyl5|~PhCoEvI?|57`82@6G>!8jFOz=A*U3B= zjPo+Dg^_;D8RH55ME^G1*&*Y$SRvySey>8%@8>_4QxwuIwUrO3e*l^8WT$r72}oo9 zi`a8nB_t^?6$l=M zS7*_Wa%@{?)lt=1ESLmzDpEi28b$i>4uK=Eq3s7wR!VPH62X8o=S@kqi$ zdx^APG(B@tej64q%^B84o1&dJ)O26hgj=?#EqE5QX}n<)#U>>M{M+L#=(=^W+4|54!~vQji@C4uApkN;AnErSBt zLF`$E36?WRH}=zz$|@&6_r=CM6T^Op&n9TOOVzJIg$|)stX{LH{X3X-dSIC-do1z4 zm}`CqDj2QF6b8e2>ZmOe=s@I9PfTftZUkbP2$~|0-yh5QNe&3p-YsM?j1xAKy_O6i z$rv0I6UbSe?=Z-Z6|bG#$PB>QsSXlCADLMITjMO)XQN5fd!hlvF9=O2p}2w%GQJVm z7F}Q?&SR2W>T&|(6MU>mB%1{Ko}Gf_2%H=$={e$x6-;ImJfqUnqzPXXs`LI8(4&7V zQDGIFspq`NTz#GqR(`E_mFY02{t5;vhA(kVKn9oi+gx^u%gSlm)aIFpU>y9mo&Y*q zra<)@aV0hvT90EhOyhE#cp?&@NL3l&$s*=YxH6>byb|yHl^i2Vr`YKR?s_Jh@6_`nwL|k#H8o0d4i4(xuTwAgozpr@hln2w! z57?i_2HNvB#&6%OItl^LOudC2qhAHK6*R0&I?A~cUrsv8Q_D3RbwTs$pE8{OAHtWR zHTyjTW^HG*=nkj5I^9XTPk#k%#>vfQ^msjuLvh>&tqsGDn%rl?L1ytJvP=S}#ii5c zl|qs))|&?wY!O|z9FEViLbPP*lTOD(Yy%coT%3_-b+Hv)S$al)OIS%z`j}=KM(#;J zuJ=2#-y7G|q(3xG<9=Lkc4R-*e!@vVs50Lh|7jDgANF2ZB>g-uKJKh=?$fqJgaTbQ z7)ygnR+%%3lG$o9PW|`{ghrOS7|=0^EbiBvP+^*tpf^Y?sYX@9VGn8FH8x#PL>a!= zRKc!IJ2Y~5wY**hE=BlKcwYYb_@0XGZ~#1wuBJ^dZ2X(YS!5VYHW)A{`k|Cfc-f+s zlAvBPyR60ZeB7pmG*1Yd(!$PQ>=TtJ@a8^Ks3J46czQ9isWH-P;t`_o6p)*Lvx;T# zM+3(#2y+~DGJ)yBX2Fj^FAB$OShUYfgsMZ>DITql12ZXrvPzra*ItPBeFV$Jel&cf zQ6U)DA?!N1)Y}^^R4Fz+nN5jSGVT&k8&*k5dep`4uD>kcB zdMlRwal-55HZBgU)1RlGHqt%~5QYRylTveq!4~mGXhx$S&2`}q_fK~o(Cn%&qf>fF zbO`+(@wRhW{ATPfZ7}aO%;`^MP;>y#)6C{H{P?DzFk;WDk{oLzU=vDIJIG4V>ff}DR3_eRJj34_m47);RTnBUKj$O8%<29V0}mBr4(h0#uS9kf z!n;ib@~tJr{uajzNH=F9^$G2R=Pm;pJ@?P9`JY?)L_d*I(G3NSXKV5)!r5HCOZ0(K z0UoyI<^?YviU8gj=jtVU!zhC7LpMEX*w#@V{Mey5+nXG`vEv*KFJ_djT-vMHK<1@O zkXGw+`5E2{sIU9Wi*C(LTp-KZ#r5{q%l656t~?o*e`HYyPjRKEpx$E>q>lsJ=`};m ziO=Rtk=v3_d)Ce9vUi;AaZp?O&1ko`{_1@>(S!bJfss?gH3^LSk8qWPgu3`$IIo*r zA7+9#N=5RP(8SH2Qw3~-7z=3_lt?KE4zZu5^0jjIGiDx}OP)n{CyCUS8t2W}p9+RG zg`Tdm^c)UaCQ40O91^Z*graA}0$1@>oQ%I*Td;7RT>P5X&H|6Kk7h0cm+{J-t2hpL z<4ub!2zR%@+DKf)liON_m%8<3FNkg(UXGUqNJemZ$en(Imjy`%WN^AgGPqsiL2Mqk zq;R^Gzqp$9KQR9njzA}Y&$f&QnQa{QZ~W}}aJw5N3XGBljtEF$GebV=BH!sG@xf)k zdc;ZOeUN&84_(O@C34-+)IZ!xWKhJS|J9vD3vfpJjsI0@fX3YC~%r=)uL zCr_OopyYL+Wre=oJ=_&SwK9slP;9Sr#{r{ggO`QaamXkrP-;Cm2&1g_^Ol|0JFa@Ka+{C(iu`~^xmtM*yvU7}hyYyuTzMtqqC0qHrYmgs8!=_}H&9Vk&S&d?GSEo#Z>m`${kb~q zwdvw~vkUvpIWR5%Nk2dBU#-6fSTR%C`@5G`KK9=TmcJe4tjiwF|8oY*9y#Y-&n@4a zwbjN;-ved3n9Wgz-@%dHqS9RljQ3Aa%lc)JwMxUQ75f*;cg?jjD0;2yzhJpEvKC4A zF&J%Yk+ka+lFoA=D)+XKtm`zPR>pnad?4z&7L_tDohR@0LH9bP-B*>o#ucPmheMh5WYN z(T4!;1t6m#I!gkX;=^t0^W78rhgp(~j3S}bO6h-xF>-4yPeV<=c zqTPhbNF#s)wMiTT#Pzx*2Q9<|pIyA9#r5*}y`-(4)ar8AcNX+fljtH8B(wUd)AiFF zl}xdsxLxjuhvIm>OK(aB6f8dEA6q1bYh^wrt*J}ccgsXiPA?t=@`uW~N-5Tw05zoJ z99gniC{Tvc9%tu~NUfw-i*t&lb!i*-MEu;R@w*8uFy19?r5{FHK29Qy?z4EMluA}h z+wZBC*4O0H9)(hT?s?Sxxt^!P`od{n7}bVb({42V##joQo>O?Rup6a&OqW>SOd!=s z+e8Fp#qGqCi^r#;UWgVyLz})?0il$@f|;-e+JP50DKgx=QS4^6wbqXk;~i&w-u0zv zD+{)`xYZQ`(?kK9Yki2Tp6xSh>?!}8g})X^OOeB+y#Tn?&bHwDAG@(y`#YMoE}}_< ziV9KB#$zjZ3ubV(oHv@gJJ~RDwL#T-9iki!1d2;!^a6@kyL3pgD&J7xJ{)l#)_v)bsrJIlUq=xqE08N;?QT#afa+rJ1!zhtSLhZik?oEl(MB?!5r1$HbEo?~v zem{g^EcTFfHNW0gb5(s9Qz zq8~Qi&)ev6t-T4>to230TtO13#Bp?)upS`Uah|Fp@2?_;(Zo;w*GIbNSW%}NMrMdt z`}XntO5#GV03zU(s>3vZXg1cTG7wArmr{G%t9Rkz9kt9!%X}a!UxF{YIN)-76ww(J z^Xm5Tc$#n6w*X^*m~0^H+GEd{tx=z=| zY;MHXL-p8U4quj#ex0*(IWvp9d@5lC$e{YpCm&6FH70?Q?K`XXYH4o?E5*mc3H{yTM_b` z-t+T)I&_o!Ui~^)mCNP+9cr!pzWu(6;#(y9`2ePRz&E$oZcvJVtkAskV>^^Wapz;Ph#fA)b62xuw8LR2`>Zzy%&FqX-vqDW#X$rT*SbhJ^A zV<-_8#gr{gwtp$enDXa(`9?p@O40arYQ{*m_m1_jm+f_Bv&i=A^6Vnv7=SU4m&G#0 zX4HGqGk{d3Bx*Df4bRMYdB=^;-t40YSnGC5wB5Y?JDNZ&x!yVTHSUp~Sd|&IrSKNt8I%uW3iw5#2u7RGNHjm#KfL1$zAYkY>CupUm_ov0ZG=ZQRx|`tc2b2 z4mxnIIr&l}aa+oi&J+lt1ccAKeXsoR;BjHmUfan?&G;PdZ;!_xuG+lzsA~nw);9~C zJLAQwMO=KLm8+?(v8>6_4QDX#=zTDtP{d3Ud`nZRF{TlV-ypi6EOr0H`=a-xu4hyJiRH#!3P4nEC&cbp^MI74R6u{LkG z!23iO1GDn!U;v~1A8x%E`$@?-@mz;`MQE$J&umBA*xn+mVlTY`T1h|>`!vHK6=P)& zTP4Ty*_=g8;%#^~>q%J)$IhC|nXTM}>vnD|j@R?aGNKAzDdJr=rAf?wOy{1kRe}t~ zdUkQY?%BeorZVAu!I-&VKhdtYIaxd1N1@u>-@Ob<7o$ejHw48#6 zi#+M`L@S$*g{OzL~{m|_*9@n=k*2Hu6G8p^~Y$~*1v{LOAgGCKT-6K%6r@hY2m zduMy``(eNr3NcZC$oPl8i+0OgNL?b4a%N|0$+@r;;P8IgK)Sv|yp2wQPN~J*aaR^4 zG#vXbt#&*{c@UGox`|pwKz2%F4)UbO&13a+_Dk2a_Qq#McnMk6+%7$DWdREy5JU>< zvtjLqu@Ox-1B#sOK3q;LPitWn+Z-VDmv}DTwx5YNVQl=VRZ@QdH318=Y%^lQZRODy zx9wJk`U$ST5_zD<7R*J&w9Yw(E&V;$5{+8x%D%AVFz7&w;O1~7m4 zRt-GInO*>azAysw4J2*_3*&#v2JzY(!oI0wOJSxh2Z9z!VjKjr*sj|*Y}~P2!TR|h z`_TGh|AUt;w5$v0ZwA8hYAQh=-ceG^ZIzw7uCcUO0@cgIVhY117*epSONm?E ziqPV5Ua{UC_W3caX^|A@xm4lyF+Vr#E5ns8vvE-2Sw2}GE!=@_JBz~Hsrer<5V#{9 z`Ub`nugi>DfGdyy5Dw~j3{Pt)?s;x9is4A(?M;EYG>)XdW@Rjq{a?o#`h4#$Aqi)U ztc*RR)ebhlmPG7Ibn!o2>aAbJrqVIUGNTJyx{r^ae4cqm#A8sh%^X!9*=HZjOJo=% z(mFa|A@c{&AI1GxGWgp0kB5(pPKuPia;A21*WK}@T4Yg(wF`)*Ro3UspOz+a|$#< z!2%gLe?OmRhHDZ2>U5B9$Qsye^fLJ6Z>IgR9yHBDJHXvtEU4qzD>rZT3~G0>-?%H z9Sp$qIp`6E7nV{%iY{?gEBZ<E{5j#DRupMzr5IQ(o%AF{EUFI zZv#)C=%2{E3O+x@y5S#DsqSNuKRey%28--?YTiB*ee3=*Oy5o9y8Ev<~?(>+$-ec=;oaamVCx!Y7Zg8y5?MNB>@q<=|@6T_@bY3jhJn|K2|NddAB z17F>DB}}ekn9-JKhBh1~3|+RQMQTXAk-wW4*^tK*X$r zZXGxl(Xu_yWV3nuK2XbGI>k1?bd^^wBF!#<(aliy>qUox&jSyrMuB#^?;Tqc^IjlA z%@(<*2_ZSR3?iV{Hn(|grFC-G6lYY zdU~?zJou|`S$Gr02u2GG%^wfV>7rGzSKhB`ksQaO?pd?J@_^+Y$KmK}zzm1Fiud>l`j~mjvDGxW;EU}vakEz_Ngyyg- zt^8RjST=@d&VP0|$|uBMX;r*r-5=aJIJXBgv7S7@H4X}YhUMh`p|{0c3#^N?Ga*!V zw~)%sX|PjM4Jvk{rAwUt@WhNHc&|0cs}OzutbIHmvpQPMzG>wFl_*EP*hVDt+_1Y` z)QH~yTX!F!mB?M%B31!*4I(bI^@nCAVXE+cp~(h*MB)!BHl8(J`k6{U2eXIOJ$d6Q z{)=VAp-Zhe$}E3VE%5f-11zRXaU1*mivM5;E);8OT8$Qj=3 z%jBT1y>hvspR*e;i%%g+I|kDh+!OYIKd|y*zQSw1c`HfC)@p^G!!>!~fJg6tCoFEo z5Bs(DxSLk2Xz2SjkHWENDy_}KCiGhuB!&C;xTASuUlt-ZcTXoWE(-h zjF~qKbBAQ{2s3M-Ob<<*3V4;?TMade+OI?`1l8>@t-!;Dw)F!DvfOWZXM~< zrA5*h>~u<2rFLCkd|+Q(b+VlSSn72O9kXg^_OVC*fyotofY2J%kgCJ)I5x~+zQ?=W z+RL{gn%;#&kDDT}BevmxK2l%})a5WkNJn11`6fbZkkkW7!n@iSEP&Qnn{-Xi31+s3 zGmV18pa74w5ro3sNYoY-W8D(^m4m}dvfWv+kAPIHP*Eb5-BkN{i~+As300A&a8vZt zWH4hK5Y;R2NEp@sRkSv7GMHiAFxQq}^P74}Rx?wtrL@@ev)#tqN~v$?E!DG+GL$7~ zX`{nX=pToAJ|G-vUIvgyZ2dVEkIKpVA;mXXLifTK1c{#U&!AY{%m~5{2e{dCADa)- zQ%idF2?=f`3?9hjzYvT%J}E8-swbIDUN;$*)m^#4lhyT^hwnz2&%}FbDs*XroT;g7 zgfU726s7Glac?HnOhs5>EGb?%-KjX>(Sm*jsJU@=kQ-ooi9aw`hDcbl-wYWK9%svD z5wz0LEs0s%6;>WP7Xm;XlsWlC&l$Jxt#t?0wVbA^Xkj;Y`mohl)};ow*AoZCVva-= z;Z}t3b-ixNm920qJ0R>N++By(MP}jjSIIv37%J@=ZhfaE7h<7Mo4&)0RESY0-?|Me z5Xl%QXk#jDgNuGR`s)@If69gDOdQkwP>Fl zPqE5C1kH=96t%V2VdLf?G#uV%9!1-*1UWUT?`Yw~`tFgq1)3*3G@y_-yD?t3zb8jVE@2{2W<_E+QJ*FJAw?yzwQR-5$!d;WqcU+b;i3$i0FzR*jsd}3pu$eiAJ z0EIZ19EE9aEKiq~MpzCi88261ES79>G!f4DB6q7c@iK>>_POXE)zVWmQSq386F!>DBa4F6;Y=clk2*d`E4s(cQ4)OJ^V z(lj)QVuS3D2p+q3^~9GOI=aW`_#1Umw#L@D6~_ zgy46OpIk#N!Co%m!r%{2a=XbT}_08nY7z6HW(utY0*fsV;i=oj= z>$EmZwt#ryTz<(pcgnV2e9;B4PUY#mKHFVn%jcr8ZuloScu z1zPcsS>>w!j_c+>s%q9_!C>&{J%C#(2mnXYn_vMJ#E)t{c0l%5UMj!`l+dc%=Pb0Oi|k%JJWY| z1pb_pR40)pDu7UaZV7II4$KGXx_y)nVwzLmoo`b$)vJ zD-G?aU!Mba8k(^TmE%FE(MB^zca9omj}|jswWUcoj3y@#Ft#Iq;<>oFkPr|Bpa0DT zg}&!BUxl93SfEk(gWtzmxD0v@nmg|?H=fqzh{cp-(vHxzo;!eh@E}8y!aKc&LnX_i zCtES0+GUuLurGI(DHp&EnB}B;+*?W3>6Z>VY|sh4&-72TK6$DYMaW{&Ba7>zx;T(7 zpSi|vU;A<(o{6nluiEWnZ4Q(6()(%9jhY|#@<^aE=0Q=a_7Ma^>tFiY2fe~vfGc`~ zmtUmhi9qZJwJMJ`5AD4yEFfXr%E!QPfOdCR52RX3K)3S2BFED9SS&+SMigjcz<|pf zG}4 z!aJ&^Ug{v6cex(b`2%~#`7 z13G#}$3qy$;!Ucsjthq6azV8ot2$*wT8jA_fw+&(p|vuDHy$0H>h0F-e(n%%J~TX^ zYI(+Bwm8P2Le8oNcd!Wmp=2vo~< z2f(pQ#_~(LIszLpxUcg-EBE2`w0p5Qnmw2JQ{~$R6$R`W@pCp^*i8Sbr+~0lRM;l6kat+-blqS5;m{9b9Gvxid^= z!`}BjeohX-XynJLLb~>9<^6GBw$pW=20mQzhYjSuj(UF%{AQ=zexCG3(|d38{Ib^E z^ZOxcKIT2~r1)(1g5(>m*ViOzF>i4t367fi3f!u;nm&Pe#!yl*JPYhylU@LJ9cP50 zzH;Jm2c|Or`uY}Y!|UewyDvG=o|4@R*ojbjCvy94B!UxMH`wsihtIYb4&aUNXt3?{ zTiM5F5W)8jKv9Si`MTduM(?LEaW*_Kc_W}bb*AyblQL#BcJ&uE+cK$TnCtUZffLVL z9!}_xFX1o_a*fg4k5q_ZtHSk#B7E8Q#g6lLJ{;mWBe%g*X`0w;0_4Tg^?T~5_hy=? zv_3ClbM4e%a9pp1)kxHe#|i(>I0)t8C{H{4M;t>u8%^!LgdjwLc#AQtO{y zlF@406)qM8x4)OZz}DxA6+@YR=ekVJf^?dSRE`z11FLflvAL2GXZ{u4fiGSx>p1V& z*}!Ktt|>3|b3EL@sb7r3T89Bqrx7Yu2}k?Y=$ISmY)7wv&g|l} zpNiVUi0T`+s(m6v=47B)U!ZH(lYG$ou61Ag1f+Xr6o@Hjqjor*AzjnCs@~xG$Jd+K z=nQcewf2@6Vy3_YT@svwqQdod`od=LyZ)Y7$8#p#Vy^>QB2o_0({u7 zYF!Ro|CJ`5-MrWynG`W6x)q155|4iDYt`3H#Juz+c0MWuVx;y2+&HIch+_&N*OY%E z%K*e)WS_?jH^|S!8ENB#NnEt5``l|kf$#Q{3M~#sADFy#v>hO{+QADjbwL5n|7nku zX@WIeWB2xcJpCjmeK*X$uxavA$@L0#;#Q=wMegC_^l*(i(?h6C+gH)h32h*%7*{9( zbNC4s<%kc;CK;I$$Y9>bw$DM-YOTXl4nfW2JDjSm21)N{H%qtGNy91q@(W_Bg8dvr zW>%Hw6IG<&K*p0N31}z(gu1P~5qnP>ZOcjHH64-{m(j>64%g8tz}w#$#H!Urri)|? zzqwu8RE-1un8mQdf128TKU0HR^x9Q0!SNeOTV3M-N)$iC0z_jj_*gEIA1zDTQqtd^ z#YwapLYJnCH)U(Lw{Fpxgm*?FxsCBO$|6>ikxOnsgC-gEIYe#b?bP>lotcFsw>)N7 zUhV)1UhQXj_OccEpC347QCVCR)_;POI1Qnb`EkssDyzId`r-V*)I!fae2U2_#D`Ej zmNMt3A$4bP0Rb*Av7%Ik!oB|rzIlIZbwPJzx_bmb7W9lU5$hp*DjfAgUm#YSdAr|z z@6Yj;wK%UtdqDEoF^LdR!iAwUD$Y@?gLx7aeBcS_FK^cZTM=4@QGn@(gJ>pUm~v) zDhsqqgVoq!u|u=qE6#sHb}LEih>2O<#S#KBj6Xa;8FU3ADy^z5eHLC67qiz3H2vW1iH13ZDdmEpfetK9 z!dVSB!KR@6f#3OQ>P+hR6W*eUhHu4x%+=d>=lXmgQgw~H28a7p#ivm1Yttly30h19 zeSvu!o~YHr)9qvX_k!yY)UM@DNnV zn>oV_1ZXWPH`mWp^>GO@boAQIb*;jqB$JFhLxTF3mj2pxcvXTp@J|_rb+o>G=FJ;q zV!@`qJ(+eS(tT0Yl%K9I0KQlwEiLgP-vpGegka=)L_&+~(~NbKg*IyXWI0oK|F&?Q z2yDl*Q+u66fU9O3Y1+VM(pUtutzZ{XEX3%(_L5*E#i?(jub9M@{Ft~wb&}TZp%co9n~pQV!mVYFcO5l+D(N)U7+*qqj#6%BWFz z&=OLdXC*V5v}c?q6j~ab#{sud_dR2inq_a{>tnu=V?t;6a_gf>&v(T>?zv=F2NECd z3%n!$7roE^4r$Hv?yJxuQck7g_0c@OVva%}4QTOZkDt=WA|XLG<`fot#31ZvPgDI> zld-fEZ~=vV=$&j&Gj6`?C=^?m)V}u;B-Q(}Rik)oEHwS8R5c=R>JWDM0coe?`HnIF zoz*W~o!FNv7WzCii_L{53M;p(VJF?9+P_w~)&f|7fI|QVMpJ#`C(umN5+uea)OF#WrxLv{?P`I5yC=k(%x-gbuBsXB8(o zqN>~k0WrC#OV!G}f5oc+mE$Jk|JL-rktpbCwLB+4Q#%QA(oHOVy931k-dGgy+=5IS zS>@m{+q#n^WNokr(F-~gt{(8}0~i6FG3!QHLGf^3mMm63T~U6<;*O7^<<@>{!r*^g zaoVI?4qquvT}&ASg7d@6_vHgy=p%JM=OEF$M$mTnF}II- zyOIAOsoG@D3&F}^ZdhC+99%gssH;O7TFJJ7<7G(2)VohkYG40Ygat3m&VO_TOlN^$ zy5lBOVgXR4B#&2`s4o2;8M)yT5r& zx8~NP5E*g)?|6b7%rG(DJ@t~>C#%K=`6gz6k_38F5*TpFX9$a?@^q|(H5J{+zZ)bY zK&?h;7K_$pUEE)3@tSu?N5~+J^NNZ%3-aGa-IegI{)t7?}YmNDY*x01DqE$U-g zW@nAe=Y;O{ecJ_;)l}y&mI_S{&g1tu{SA0&t!ULFyxMXW#ebbt^?RLz0vn>4{dn}a zT2;LNttPUjbbeMRswRB;h<(csLtbzO99$9~{?#+2r#{fna^JGk`I4j;Mxty1p=N8N zgQbu!|9|R-<&Tmc|64!&wVxrdpkCob&|1@mr_?U$w;09P@|x;Y-HbxryttYO&UE{d z9{w5&huX81&&Mp=5q#}fzQ(6%U?pPuR8n?{p)LD*Ks5@*fcw6Kh^dxpLG|m?T$8GW zGf%Kz>|oU3Qdm>-wq{AfowBl)4oF$3*gWzw*xb%lWa`P{-r4O~$8mVRUg+=O1l8}- z{TURgi&?_KV?d9n{y-Rhc9U-3ZJ`V{434ugncBsut-+~G*pe~py#-`uHCd8}t3k@j z%}d>|ARP`AyMzqSU@u#vL6eeSm83jPq~K}HKVYJb;~T+tOc;06se`IUjo{AEcICU! z1fnS|W04X~uPe;t_A_-?kPO*lu$LI&p08dML|FG;EDtDEF`wOVZZpAeuJ>Hq2)=Ne)uLk16(@O{@uQmM@-l_Ck8Ct z0~8yMWyRH`T7<|Leuu;KgwasRUd|G^=4j=}{4j&=u&W*1W4jMIE_hOFe*qPXcYI!p zXg@40F#vvHo67us;JI-TahU8VwcqSJdknjf*A zQ23tsMnFi=*^F)lrmOipR;C(KAF4ipvtYKIZ-yr*9lBS`_H$z3#G$k~f-eG{p&MDo zc0Df?HfiC2lzf{Wo3v@cHl5@jdt=)m#?zcEQ5kt?$x|5s5PdGqsCs+^&xY2K(v$Hr{l;*c0UKaR#x?H2W*VH zuOMrz7QPc@kq7r@P2vu%A!XrtYCK+K9tn0qm)HJI3Rds^Gyhmcq&WF8461$j-&GQwJGx&}l#q{+dcTBGUyT;IvFHwnA;JXcw&Z z_k|R&L8IWh#N(zEgBXLpzJFrb70EqY@S7j^=}%y6aOYDZ|0>=C8kPs-4b&@K=+ z7K~k6Ne81u>S8h<6hgLVcfoI2M+$+kmsYIk;8>4egl>1GnPBY+!qvD{G|fgaZvyCc zND)sF@eHFzWL*?1oJSqyxi)9x2Ap)gBvC6#(%a*quncTZBae7r^~U)K*Ht5G;B}N08!S2F+y?)q7sxN;pnp1#dr!m` z*MD|dc_h_m*&TH1wMx=meeT%-O@-`^s6uu|RY~+H?LD5Iygc1r(q~}@Hyk0GSQuLK zboGb#=!&Ok9Y=GZEOJdYg8#n_BbOIRq&e&`Ja4&JlK1Z~FZ7I%?-Ltq*F$APJf^j`Wx`ecJc@&z z4ARw4N#j6WZsnu982au7G;i0|=WXBBK$lE}A>_sxvt}Ys%{ZF8!>pOjPbjGsTr0Gb-f2i?Ue%p6X*FEldLG5 z4*BQ5j;7P;QA>FyzVM6FyXQxs)RJ?E!*D*bi#iVD$R*pDKBy3~qAjqkX6H&uD1ATMuna&%nqDow5kd=f~A*Qq1t9iSb}W#(+=1n z6o5UK#inc{kxj|pwG{7YwR%Bjuub~O{>YK5+6g>Z9KGe#Z4vX)aWUV}<)Ga+@RxX- z_~zYnTC7*}xm#Y@ij}k>M6Q`Qdw2QM7uUT%gmvn$<)#O=+~i3?{Ev-ZwajL@9ZQ{f zUPJFvFD#Z^0HiW(DUUzp+H5bSUA##q#T`>d?91d*@PE|Z)g?adNdfaKXn|_jFyxF1 z&t+TEt*zBXHlo!<5bXB{sTgM(}ET3LB5lHJy`X{{&AipUJB`eJ^0nlmvhnWRXq9?1FL|W_H;ZYH3{go>Pv>yl znQOa7ksG{oDabi3S#eG^`}osoW=&7c6s1pS`#c|h;6shHCvyAg=Q+ZyN^hCKd1%qP z+RSH}EKe83=0ldB!s$0vY=J=F)8+!{^$;`EEqSHbT83%}_>xGQ0TVwjIzl zI+)9{m15QQg}~>nNT7!wUusFtNx!gp!Njs!!9Y0{&Ns3ynX~FXC}zXm8lvqHDhsY< z!L#vL%*i`=58`oZfa|U=EZk3SzZz;a3p(W|54dUpY=fl|;JKCxq(EnlNwx_aJTH z(jfmApqQgA8Uou5#QX2Geik)yCfgNLVK8GxMa{~)@I>|y;8^(+Q+9Jw^C=Y-&+3GE zMa7cURe{3>pC#o&SM_v9f|stXZa5b;TCr zm&mnym1ISgrm)R{-Q`h{@Cl)f%-(mx4zu)COwnbtdZCmI41(}4_QO~8&2{8eo&!+~u z2Wn{4;QDN_Fjw~-E4DSrE*k&x4V*o5n%&-0m1~m8a!o5f=LN6AZWkUqe^9otqK2z8 zB6LMX*1}l>XBx~U75G-LMbx~|u&W|h7A-(3$(S;$v)Gb*CB-tvrMDMvUy!prw{TG# z%@&&i`KD(rP5%TG#uKvP`G1F-^Qc8lexuQha+ztP6lH8(=~H>f)E#H{r}t9bar*fYo&FB}f+G}MD7nThVa-%8?DF~v<(zQ%2?1 zM%xyC=HcmvCP4qBT~@$;a7fYhx&V=!78r7`D+IRk?~S6``6s=;<2E;jq-ZImrRd4kl(akcH_0CvTdU zTLw(T>9PvDOC3ZvX3#QgPWX8Mv(0xapP+2K;6=q0%OOSrY*Oa`TcG*j7SX1<`vqG2 zH4~=@q_M7p-~3XK+y*t_aCRL`8f5w*fejx&M&Prj=hHk8*D%dgetPxt?Ba))voe41 zKAt^$_U!8~zl8rjd-kmV-)CPu|N67%Uw-q|S6@DV_QjW9eD>`5=U;sD^=IVSgO9}X zY>W!`*|UTD>J#@Xd46X@&(5zd$ZJ-V$}TWbDOx~yg1t}vlFN+Tv$^IK``dpw8wZBuC@QiQYE=I@ e`_$ZgdY+!A=jr+NJpVrc0RR7oXCs>c6a@gWFG#Nd literal 0 HcmV?d00001 diff --git a/charts/fleet-crd/104.1.5+up0.10.9/Chart.yaml b/charts/fleet-crd/104.1.5+up0.10.9/Chart.yaml new file mode 100644 index 0000000000..93b9e9e9ed --- /dev/null +++ b/charts/fleet-crd/104.1.5+up0.10.9/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd +apiVersion: v2 +appVersion: 0.10.9 +description: Fleet Manager CustomResourceDefinitions +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-crd +version: 104.1.5+up0.10.9 diff --git a/charts/fleet-crd/104.1.5+up0.10.9/README.md b/charts/fleet-crd/104.1.5+up0.10.9/README.md new file mode 100644 index 0000000000..2452ab2f1f --- /dev/null +++ b/charts/fleet-crd/104.1.5+up0.10.9/README.md @@ -0,0 +1,5 @@ +# Fleet CRD Helm Chart + +Fleet Manager CustomResourceDefinitions Helm chart is a requirement for the Fleet Helm Chart. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-crd/104.1.5+up0.10.9/templates/crds.yaml b/charts/fleet-crd/104.1.5+up0.10.9/templates/crds.yaml new file mode 100644 index 0000000000..4ee4cd986b --- /dev/null +++ b/charts/fleet-crd/104.1.5+up0.10.9/templates/crds.yaml @@ -0,0 +1,7076 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: bundledeployments.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleDeployment + listKind: BundleDeploymentList + plural: bundledeployments + singular: bundledeployment + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.deployed + name: Deployed + type: string + - jsonPath: .status.display.monitored + name: Monitored + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'BundleDeployment is used internally by Fleet and should not + be used directly. + + When a Bundle is deployed to a cluster an instance of a Bundle is called + a + + BundleDeployment. A BundleDeployment represents the state of that Bundle + on + + a specific cluster with its cluster-specific customizations. The Fleet + agent + + is only aware of BundleDeployment resources that are created for the cluster + + the agent is managing.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + type: object + nullable: true + type: array + deploymentID: + description: DeploymentID is the ID of the currently applied deployment. + nullable: true + type: string + ociContents: + description: OCIContents is true when this deployment's contents + is stored in an oci registry + type: boolean + options: + description: Options are the deployment options, that are currently + applied. + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + deleteNamespace: + description: DeleteNamespace can be used to delete the deployed + namespace when removing the bundle + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + format: int64 + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disableDependencyUpdate: + description: DisableDependencyUpdate allows skipping chart + dependencies update + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + type: string + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + type: string + nullable: true + type: array + type: object + type: object + paused: + description: 'Paused if set to true, will stop any BundleDeployments + from being + + updated. If true, BundleDeployments will be marked as out of sync + + when changes are detected.' + type: boolean + stagedDeploymentID: + description: StagedDeploymentID is the ID of the staged deployment. + nullable: true + type: string + stagedOptions: + description: 'StagedOptions are the deployment options, that are + staged for + + the next deployment.' + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + deleteNamespace: + description: DeleteNamespace can be used to delete the deployed + namespace when removing the bundle + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + format: int64 + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disableDependencyUpdate: + description: DisableDependencyUpdate allows skipping chart + dependencies update + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + type: string + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + type: string + nullable: true + type: array + type: object + type: object + type: object + status: + properties: + appliedDeploymentID: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + nullable: true + type: array + display: + nullable: true + properties: + deployed: + nullable: true + type: string + monitored: + nullable: true + type: string + state: + nullable: true + type: string + type: object + modifiedStatus: + items: + description: 'ModifiedStatus is used to report the status of a + resource that is modified. + + It indicates if the modification was a create, a delete or a + patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + nonModified: + type: boolean + nonReadyStatus: + items: + description: NonReadyStatus is used to report the status of a + resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID values, including + UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names do not get + conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + ready: + type: boolean + release: + nullable: true + type: string + resources: + description: 'Resources lists the metadata of resources that were + deployed + + according to the helm release history.' + items: + description: BundleDeploymentResource contains the metadata of + a deployed resource. + properties: + apiVersion: + nullable: true + type: string + createdAt: + format: date-time + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + nullable: true + type: array + syncGeneration: + format: int64 + nullable: true + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: bundlenamespacemappings.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleNamespaceMapping + listKind: BundleNamespaceMappingList + plural: bundlenamespacemappings + singular: bundlenamespacemapping + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BundleNamespaceMapping maps bundles to clusters in other namespaces. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + bundleSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + namespaceSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: bundles.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Bundle + listKind: BundleList + plural: bundles + singular: bundle + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Bundle contains the resources of an application and its deployment + options. + + It will be deployed as a Helm chart to target clusters. + + + + When a GitRepo is scanned it will produce one or more bundles. Bundles + are + + a collection of resources that get deployed to one or more cluster(s). + Bundle is the + + fundamental deployment unit used in Fleet. The contents of a Bundle may + be + + Kubernetes manifests, Kustomize configuration, or Helm charts. Regardless + + of the source the contents are dynamically rendered into a Helm chart + by + + the agent and installed into the downstream cluster as a Helm release.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + contentsId: + description: ContentsID stores the contents id when deploying contents + using an OCI registry. + nullable: true + type: string + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or lock + down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this will + also delete all your Custom Resources. + type: boolean + deleteNamespace: + description: DeleteNamespace can be used to delete the deployed + namespace when removing the bundle + type: boolean + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + type: object + nullable: true + type: array + diff: + description: Diff can be used to ignore the modified state of objects + which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove fields + from the check for modifications. + items: + description: ComparePatch matches a resource and removes fields + from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain JSON + path. + items: + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the resource. + items: + description: Operation of a ComparePatch, usually "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + format: int64 + type: integer + helm: + description: Helm options for the deployment, like the chart name, + repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is performing + an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI registry + based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's EnableDNS + option, which Fleet sets to `true` by default. + type: boolean + disableDependencyUpdate: + description: DisableDependencyUpdate allows skipping chart dependencies + update + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. This + could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to deploy + the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to download + the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema validation + against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for its + own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values from. + items: + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps and + secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as ready. + It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when monitoring + the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + type: string + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed resources + when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the dir + containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize resources. + This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + type: string + description: NamespaceAnnotations are annotations that will be appended + to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + type: string + description: NamespaceLabels are labels that will be appended to + the namespace created by Fleet. + nullable: true + type: object + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. It will be marked as out of sync. + type: boolean + resources: + description: 'Resources contains the resources that were read from + the bundle''s + + path. This includes the content of downloaded helm charts.' + items: + description: BundleResource represents the content of a single + resource from the bundle, like a YAML manifest. + properties: + content: + description: The content of the resource, can be compressed. + nullable: true + type: string + encoding: + description: Encoding is either empty or "base64+gz". + nullable: true + type: string + name: + description: Name of the resource, can include the bundle's + internal path. + nullable: true + type: string + type: object + nullable: true + type: array + rolloutStrategy: + description: 'RolloutStrategy controls the rollout of bundles, by + defining + + partitions, canaries and percentages for cluster availability.' + nullable: true + properties: + autoPartitionSize: + anyOf: + - type: integer + - type: string + description: 'A number or percentage of how to automatically + partition clusters if no + + specific partitioning strategy is configured. + + default: 25%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'A number or percentage of clusters that can be + unavailable during an update + + of a bundle. This follows the same basic approach as a deployment + rollout + + strategy. Once the number of clusters meets unavailable state + update will be + + paused. Default value is 100% which doesn''t take effect on + update. + + default: 100%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailablePartitions: + anyOf: + - type: integer + - type: string + description: 'A number or percentage of cluster partitions that + can be unavailable during + + an update of a bundle. + + default: 0' + nullable: true + x-kubernetes-int-or-string: true + partitions: + description: 'A list of definitions of partitions. If any target + clusters do not match + + the configuration they are added to partitions at the end + following the + + autoPartitionSize.' + items: + description: Partition defines a separate rollout strategy + for a set of clusters. + properties: + clusterGroup: + description: A cluster group name to include in this partition + type: string + clusterGroupSelector: + description: Selector matching cluster group labels to + include in this partition + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + clusterName: + description: ClusterName is the name of a cluster to include + in this partition + type: string + clusterSelector: + description: Selector matching cluster labels to include + in this partition + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'A number or percentage of clusters that + can be unavailable in this + + partition before this partition is treated as done. + + default: 10%' + x-kubernetes-int-or-string: true + name: + description: A user-friendly name given to the partition + used for Display (optional). + nullable: true + type: string + type: object + nullable: true + type: array + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this deployment. + nullable: true + type: string + targetRestrictions: + description: TargetRestrictions is an allow list, which controls + if a bundledeployment is created for a target. + items: + description: 'BundleTargetRestriction is used internally by Fleet + and should not be modified. + + It acts as an allow list, to prevent the creation of BundleDeployments + from + + Targets created by TargetCustomizations in fleet.yaml.' + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + clusterName: + nullable: true + type: string + clusterSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + name: + nullable: true + type: string + type: object + type: array + targets: + description: 'Targets refer to the clusters which will be deployed + to. + + Targets are evaluated in order and the first one to match is used.' + items: + description: 'BundleTarget declares clusters to deploy to. Fleet + will merge the + + BundleDeploymentOptions from customizations into this struct.' + properties: + clusterGroup: + description: ClusterGroup to match a specific cluster group + by name. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a selector to match cluster + groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + clusterName: + description: 'ClusterName to match a specific cluster by name + that will be + + selected' + nullable: true + type: string + clusterSelector: + description: 'ClusterSelector is a selector to match clusters. + The structure is + + the standard metav1.LabelSelector format. If clusterGroupSelector + or + + clusterGroup is specified, clusterSelector will be used + only to + + further refine the selection after clusterGroupSelector + and + + clusterGroup is evaluated.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for + resources that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + deleteNamespace: + description: DeleteNamespace can be used to delete the deployed + namespace when removing the bundle + type: boolean + diff: + description: Diff can be used to ignore the modified state + of objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the + resource to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to + match. + nullable: true + type: string + name: + description: Name is the name of the resource to + match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from + the resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + doNotDeploy: + description: DoNotDeploy if set to true, will not deploy to + this target. + type: boolean + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + format: int64 + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or + OCI registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disableDependencyUpdate: + description: DisableDependencyUpdate allows skipping chart + dependencies update + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to + specify the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from + configmap, secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with + release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo + as ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be + ignored when monitoring the Bundle. + items: + additionalProperties: + type: string + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + name: + description: 'Name of target. This value is largely for display + and logging. If + + not specified a default name of the format "target000" will + be used' + type: string + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the + deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform + this deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to + folders in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + type: string + nullable: true + type: array + type: object + type: object + type: array + yaml: + description: 'YAML options, if using raw YAML these are names that + map to + + overlays/{name} files that will be used to replace or patch a + resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml then + a file + + ./overlays/myoverlay/subdir/resource.yaml will replace the + base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + type: string + nullable: true + type: array + type: object + type: object + status: + properties: + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the bundle.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: State is a summary state for the bundle, calculated + over the non-ready resources. + nullable: true + type: string + type: object + maxNew: + description: 'MaxNew is always 50. A bundle change can only stage + 50 + + bundledeployments at a time.' + type: integer + maxUnavailable: + description: 'MaxUnavailable is the maximum number of unavailable + deployments. See + + rollout configuration.' + type: integer + maxUnavailablePartitions: + description: 'MaxUnavailablePartitions is the maximum number of + unavailable + + partitions. The rollout configuration defines a maximum number + or + + percentage of unavailable partitions.' + type: integer + newlyCreated: + description: 'NewlyCreated is the number of bundle deployments that + have been created, + + not updated.' + type: integer + observedGeneration: + description: ObservedGeneration is the current generation of the + bundle. + format: int64 + type: integer + ociReference: + description: 'OCIReference is the OCI reference used to store contents, + this is + + only for informational purposes.' + type: string + partitions: + description: PartitionStatus lists the status of each partition. + items: + description: PartitionStatus is the status of a single rollout + partition. + properties: + count: + description: Count is the number of clusters in the partition. + type: integer + maxUnavailable: + description: MaxUnavailable is the maximum number of unavailable + clusters in the partition. + type: integer + name: + description: Name is the name of the partition. + nullable: true + type: string + summary: + description: Summary is a summary state for the partition, + calculated over its non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, + but with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments + that have been deployed + + and for which all resources are ready, but where some + changes from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which + is filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information + about a bundle that is not ready for a + + given state like "ErrApplied". It contains a list + of non-ready or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, + like e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the + bundle is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for + each modified resource. + items: + description: 'ModifiedStatus is used to report + the status of a resource that is modified. + + It indicates if the modification was a create, + a delete or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for + each non-ready resource. + items: + description: NonReadyStatus is used to report + the status of a resource that is not ready. + It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique + ID values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias + to string. Being a type captures + + intent and helps make sure that UIDs and + names do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments + that have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream + agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments + that are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments + that have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, + but are waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: Unavailable is the number of unavailable clusters + in the partition. + type: integer + type: object + type: array + resourceKey: + description: 'ResourceKey lists resources, which will likely be + deployed. The + + actual list of resources on a cluster might differ, depending + on the + + helm chart, value templating, etc..' + items: + description: ResourceKey lists resources, which will likely be + deployed. + properties: + apiVersion: + description: APIVersion is the k8s api version of the resource. + nullable: true + type: string + kind: + description: Kind is the k8s api kind of the resource. + nullable: true + type: string + name: + description: Name is the name of the resource. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource. + nullable: true + type: string + type: object + nullable: true + type: array + resourcesSha256Sum: + description: ResourcesSHA256Sum corresponds to the JSON serialization + of the .Spec.Resources field + type: string + summary: + description: 'Summary contains the number of bundle deployments + in each state and + + a list of non-ready resources.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: 'Unavailable is the number of bundle deployments that + are not ready or + + where the AppliedDeploymentID in the status does not match the + + DeploymentID from the spec.' + type: integer + unavailablePartitions: + description: UnavailablePartitions is the number of unavailable + partitions. + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clustergroups.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ClusterGroup + listKind: ClusterGroupList + plural: clustergroups + singular: clustergroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: Clusters-Ready + type: string + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterGroup is a re-usable selector to target a group of clusters. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + selector: + description: Selector is a label selector, used to select clusters + for this group. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. If + the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is replaced + during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose + key field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + type: object + status: + properties: + clusterCount: + description: ClusterCount is the number of clusters in the cluster + group. + type: integer + conditions: + description: Conditions is a list of conditions and their statuses + for the cluster group. + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + nullable: true + type: string + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: 'State is a summary state for the cluster group, + showing "NotReady" if + + there are non-ready resources.' + nullable: true + type: string + type: object + nonReadyClusterCount: + description: NonReadyClusterCount is the number of clusters that + are not ready. + type: integer + nonReadyClusters: + description: NonReadyClusters is a list of cluster names that are + not ready. + items: + type: string + nullable: true + type: array + resourceCounts: + description: 'ResourceCounts contains the number of resources in + each state over + + all bundles in the cluster group.' + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundle deployments and + their resources + + in the cluster group.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusterregistrations.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistration + listKind: ClusterRegistrationList + plural: clusterregistrations + singular: clusterregistration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.clusterName + name: Cluster-Name + type: string + - jsonPath: .spec.clusterLabels + name: Labels + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistration is used internally by Fleet and should + not be used directly. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + clientID: + description: 'ClientID is a unique string that will identify the + cluster. The + + agent either uses the configured ID or the kubeSystem.UID.' + nullable: true + type: string + clientRandom: + description: 'ClientRandom is a random string that the agent generates. + When + + fleet-controller grants a registration, it creates a registration + + secret with this string in the name.' + nullable: true + type: string + clusterLabels: + additionalProperties: + type: string + description: ClusterLabels are copied to the cluster resource during + the registration. + nullable: true + type: object + type: object + status: + properties: + clusterName: + description: 'ClusterName is only set after the registration is + being processed by + + fleet-controller.' + nullable: true + type: string + granted: + description: 'Granted is set to true, if the request service account + is present + + and its token secret exists. This happens directly before creating + + the registration secret, roles and rolebindings.' + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusterregistrationtokens.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistrationToken + listKind: ClusterRegistrationTokenList + plural: clusterregistrationtokens + singular: clusterregistrationtoken + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.secretName + name: Secret-Name + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistrationToken is used by agents to register a new + cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + ttl: + description: 'TTL is the time to live for the token. It is used + to calculate the + + expiration time. If the token expires, it will be deleted.' + nullable: true + type: string + type: object + status: + properties: + expires: + description: Expires is the time when the token expires. + format: date-time + type: string + secretName: + description: SecretName is the name of the secret containing the + token. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: clusters.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.agent.lastSeen + name: Last-Seen + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Cluster corresponds to a Kubernetes cluster. Fleet deploys + bundles to targeted clusters. + + Clusters to which Fleet deploys manifests are referred to as downstream + + clusters. In the single cluster use case, the Fleet manager Kubernetes + + cluster is both the manager and downstream cluster at the same time.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + agentAffinity: + description: 'AgentAffinity overrides the default affinity for the + cluster''s agent + + deployment. If this value is nil the default affinity is used.' + nullable: true + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the + pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node matches the corresponding + matchExpressions; the + + node(s) with the highest sum are the most preferred.' + items: + description: 'An empty preferred scheduling term matches + all objects with implicit weight 0 + + (i.e. it''s a no-op). A null preferred scheduling term + matches no objects (i.e. is also a no-op).' + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to an update), + the system + + may or may not try to eventually evict the pod from its + node.' + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: 'A null or empty node selector term matches + no objects. The requirements of + + them are ANDed. + + The TopologySelectorTerm type implements a subset + of the NodeSelectorTerm.' + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: 'A label query over a set of resources, + in this case pods. + + If it''s null, this PodAffinityTerm matches + with no Pods.' + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: 'MatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used + to lookup values from the + + incoming pod labels, those key-value labels + are merged with `labelSelector` as `key in (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. + Keys that don''t exist in the incoming + + pod labels will be ignored. The default value + is empty. + + The same key is forbidden to exist in both matchLabelKeys + and labelSelector. + + Also, matchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: 'MismatchLabelKeys is a set of pod + label keys to select which pods will + + be taken into consideration. The keys are used + to lookup values from the + + incoming pod labels, those key-value labels + are merged with `labelSelector` as `key notin + (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. + Keys that don''t exist in the incoming + + pod labels will be ignored. The default value + is empty. + + The same key is forbidden to exist in both mismatchLabelKeys + and labelSelector. + + Also, mismatchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + type: string + required: + - topologyKey + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: 'A label query over a set of resources, + in this case pods. + + If it''s null, this PodAffinityTerm matches with + no Pods.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: 'MatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used to + lookup values from the + + incoming pod labels, those key-value labels are + merged with `labelSelector` as `key in (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. Keys + that don''t exist in the incoming + + pod labels will be ignored. The default value is + empty. + + The same key is forbidden to exist in both matchLabelKeys + and labelSelector. + + Also, matchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity + feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: 'MismatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used to + lookup values from the + + incoming pod labels, those key-value labels are + merged with `labelSelector` as `key notin (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. Keys + that don''t exist in the incoming + + pod labels will be ignored. The default value is + empty. + + The same key is forbidden to exist in both mismatchLabelKeys + and labelSelector. + + Also, mismatchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity + feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the anti-affinity expressions specified by this field, + but it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling anti-affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: 'A label query over a set of resources, + in this case pods. + + If it''s null, this PodAffinityTerm matches + with no Pods.' + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: 'MatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used + to lookup values from the + + incoming pod labels, those key-value labels + are merged with `labelSelector` as `key in (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. + Keys that don''t exist in the incoming + + pod labels will be ignored. The default value + is empty. + + The same key is forbidden to exist in both matchLabelKeys + and labelSelector. + + Also, matchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: 'MismatchLabelKeys is a set of pod + label keys to select which pods will + + be taken into consideration. The keys are used + to lookup values from the + + incoming pod labels, those key-value labels + are merged with `labelSelector` as `key notin + (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. + Keys that don''t exist in the incoming + + pod labels will be ignored. The default value + is empty. + + The same key is forbidden to exist in both mismatchLabelKeys + and labelSelector. + + Also, mismatchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling + MatchLabelKeysInPodAffinity feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + type: string + required: + - topologyKey + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the anti-affinity requirements specified + by this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the anti-affinity requirements specified by this field + cease to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: 'A label query over a set of resources, + in this case pods. + + If it''s null, this PodAffinityTerm matches with + no Pods.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: 'MatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used to + lookup values from the + + incoming pod labels, those key-value labels are + merged with `labelSelector` as `key in (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. Keys + that don''t exist in the incoming + + pod labels will be ignored. The default value is + empty. + + The same key is forbidden to exist in both matchLabelKeys + and labelSelector. + + Also, matchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity + feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: 'MismatchLabelKeys is a set of pod label + keys to select which pods will + + be taken into consideration. The keys are used to + lookup values from the + + incoming pod labels, those key-value labels are + merged with `labelSelector` as `key notin (value)` + + to select the group of existing pods which pods + will be taken into consideration + + for the incoming pod''s pod (anti) affinity. Keys + that don''t exist in the incoming + + pod labels will be ignored. The default value is + empty. + + The same key is forbidden to exist in both mismatchLabelKeys + and labelSelector. + + Also, mismatchLabelKeys cannot be set when labelSelector + isn''t set. + + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity + feature gate.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + agentEnvVars: + description: AgentEnvVars are extra environment variables to be + added to the agent deployment. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + + using the previously defined environment variables in the + container and + + any service environment variables. If a variable cannot + be resolved, + + the reference in the input string will be unchanged. Double + $$ are reduced + + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. + + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + + Escaped references will never be expanded, regardless of + whether the variable + + exists or not. + + Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: '' + description: 'Name of the referent. + + This field is effectively required, but due to backwards + compatibility is + + allowed to be empty. Instances of this type with + an empty value here are + + almost certainly wrong. + + TODO: Add other useful fields. apiVersion, kind, + uid? + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Drop `kubebuilder:default` when controller-gen + doesn''t need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, + + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests + + (limits.cpu, limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: '' + description: 'Name of the referent. + + This field is effectively required, but due to backwards + compatibility is + + allowed to be empty. Instances of this type with + an empty value here are + + almost certainly wrong. + + TODO: Add other useful fields. apiVersion, kind, + uid? + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Drop `kubebuilder:default` when controller-gen + doesn''t need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + nullable: true + type: array + agentNamespace: + description: AgentNamespace defaults to the system namespace, e.g. + cattle-fleet-system. + nullable: true + type: string + agentResources: + description: AgentResources sets the resources for the cluster's + agent deployment. + nullable: true + properties: + claims: + description: 'Claims lists the names of resources, defined in + spec.resourceClaims, + + that are used by this container. + + + + This is an alpha field and requires enabling the + + DynamicResourceAllocation feature gate. + + + + This field is immutable. It can only be set for containers.' + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: 'Name must match the name of one entry in + pod.spec.resourceClaims of + + the Pod where this field is used. It makes that resource + available + + inside a container.' + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. + + If Requests is omitted for a container, it defaults to Limits + if that is explicitly specified, + + otherwise to an implementation-defined value. Requests cannot + exceed Limits. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + agentTolerations: + description: AgentTolerations defines an extra set of Tolerations + to be added to the Agent deployment. + items: + description: 'The pod this Toleration is attached to tolerates + any taint that matches + + the triple using the matching operator .' + properties: + effect: + description: 'Effect indicates the taint effect to match. + Empty means match all taint effects. + + When specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute.' + type: string + key: + description: 'Key is the taint key that the toleration applies + to. Empty means match all taint keys. + + If the key is empty, operator must be Exists; this combination + means to match all values and all keys.' + type: string + operator: + description: 'Operator represents a key''s relationship to + the value. + + Valid operators are Exists and Equal. Defaults to Equal. + + Exists is equivalent to wildcard for value, so that a pod + can + + tolerate all taints of a particular category.' + type: string + tolerationSeconds: + description: 'TolerationSeconds represents the period of time + the toleration (which must be + + of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, + + it is not set, which means tolerate the taint forever (do + not evict). Zero and + + negative values will be treated as 0 (evict immediately) + by the system.' + format: int64 + type: integer + value: + description: 'Value is the taint value the toleration matches + to. + + If the operator is Exists, the value should be empty, otherwise + just a regular string.' + type: string + type: object + nullable: true + type: array + clientID: + description: 'ClientID is a unique string that will identify the + cluster. It can + + either be predefined, or generated when importing the cluster.' + nullable: true + type: string + hostNetwork: + description: 'HostNetwork sets the agent StatefulSet to use hostNetwork: + true setting. + + Allows for provisioning of network related bundles (CNI configuration).' + nullable: true + type: boolean + kubeConfigSecret: + description: 'KubeConfigSecret is the name of the secret containing + the kubeconfig for the downstream cluster. + + It can optionally contain a APIServerURL and CA to override the + + values in the fleet-controller''s configmap.' + nullable: true + type: string + kubeConfigSecretNamespace: + description: 'KubeConfigSecretNamespace is the namespace of the + secret containing the kubeconfig for the downstream cluster. + + If unset, it will be assumed the secret can be found in the namespace + that the Cluster object resides within.' + nullable: true + type: string + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. + type: boolean + privateRepoURL: + description: PrivateRepoURL prefixes the image name and overrides + a global repo URL from the agents config. + nullable: true + type: string + redeployAgentGeneration: + description: RedeployAgentGeneration can be used to force redeploying + the agent. + format: int64 + type: integer + templateValues: + description: TemplateValues defines a cluster specific mapping of + values to be sent to fleet.yaml values templating. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + properties: + agent: + description: AgentStatus contains information about the agent. + properties: + lastSeen: + description: 'LastSeen is the last time the agent checked in + to update the status + + of the cluster resource.' + format: date-time + nullable: true + type: string + namespace: + description: Namespace is the namespace of the agent deployment, + e.g. "cattle-fleet-system". + nullable: true + type: string + type: object + agentAffinityHash: + description: 'AgentAffinityHash is a hash of the agent''s affinity + configuration, + + used to detect changes.' + type: string + agentConfigChanged: + description: 'AgentConfigChanged is set to true if any of the agent + configuration + + changed, like the API server URL or CA. Setting it to true will + + trigger a re-import of the cluster.' + type: boolean + agentDeployedGeneration: + description: AgentDeployedGeneration is the generation of the agent + that is currently deployed. + format: int64 + nullable: true + type: integer + agentEnvVarsHash: + description: AgentEnvVarsHash is a hash of the agent's env vars, + used to detect changes. + nullable: true + type: string + agentHostNetwork: + description: AgentHostNetwork defines observed state of spec.hostNetwork + setting that is currently used. + nullable: true + type: boolean + agentMigrated: + description: 'AgentMigrated is always set to true after importing + a cluster. If + + false, it will trigger a migration. Old agents don''t have + + this in their status.' + type: boolean + agentNamespaceMigrated: + description: 'AgentNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents + + don''t have this in their status.' + type: boolean + agentPrivateRepoURL: + description: AgentPrivateRepoURL is the private repo URL for the + agent that is currently used. + nullable: true + type: string + agentResourcesHash: + description: 'AgentResourcesHash is a hash of the agent''s resources + configuration, + + used to detect changes.' + nullable: true + type: string + agentTLSMode: + description: 'AgentTLSMode supports two values: `system-store` and + `strict`. If set to + + `system-store`, instructs the agent to trust CA bundles from the + operating + + system''s store. If set to `strict`, then the agent shall only + connect to a + + server which uses the exact CA configured when creating/updating + the agent.' + nullable: true + type: string + agentTolerationsHash: + description: 'AgentTolerationsHash is a hash of the agent''s tolerations + + configuration, used to detect changes.' + nullable: true + type: string + apiServerCAHash: + description: APIServerCAHash is a hash of the upstream API server + CA, used to detect changes. + nullable: true + type: string + apiServerURL: + description: 'APIServerURL is the currently used URL of the API + server that the + + cluster uses to connect to upstream.' + nullable: true + type: string + cattleNamespaceMigrated: + description: 'CattleNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents, + + don''t have this in their status.' + type: boolean + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + type: array + desiredReadyGitRepos: + description: 'DesiredReadyGitRepos is the number of gitrepos for + this cluster that + + are desired to be ready.' + type: integer + display: + description: Display contains the number of ready bundles, nodes + and a summary state. + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + type: string + state: + description: State of the cluster, either one of the bundle + states, or "WaitCheckIn". + nullable: true + type: string + type: object + garbageCollectionInterval: + description: GarbageCollectionInterval determines how often agents + clean up obsolete Helm releases. + type: string + namespace: + description: 'Namespace is the cluster namespace, it contains the + clusters service + + account as well as any bundledeployments. Example: + + "cluster-fleet-local-cluster-294db1acfa77-d9ccf852678f"' + type: string + readyGitRepos: + description: ReadyGitRepos is the number of gitrepos for this cluster + that are ready. + type: integer + resourceCounts: + description: ResourceCounts is an aggregate over the GitRepoResourceCounts. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundledeployments. The + resource counts + + are copied from the gitrepo resource.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: contents.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Content + listKind: ContentList + plural: contents + singular: content + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Content is used internally by Fleet and should not be used + directly. It + + contains the resources from a bundle for a specific target cluster.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + content: + description: 'Content is a byte array, which contains the manifests + of a bundle. + + The bundle resources are copied into the bundledeployment''s content + + resource, so the downstream agent can deploy them.' + format: byte + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + sha256sum: + description: SHA256Sum of the Content field + type: string + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: gitreporestrictions.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: GitRepoRestriction + listKind: GitRepoRestrictionList + plural: gitreporestrictions + singular: gitreporestriction + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .defaultServiceAccount + name: Default-ServiceAccount + type: string + - jsonPath: .allowedServiceAccounts + name: Allowed-ServiceAccounts + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepoRestriction is a resource that can optionally be used + to restrict + + the options of GitRepos in the same namespace.' + properties: + allowedClientSecretNames: + description: AllowedClientSecretNames is a list of client secret names + that GitRepos are allowed to use. + items: + type: string + nullable: true + type: array + allowedRepoPatterns: + description: 'AllowedRepoPatterns is a list of regex patterns that restrict + the + + valid values of the Repo field of a GitRepo.' + items: + type: string + nullable: true + type: array + allowedServiceAccounts: + description: AllowedServiceAccounts is a list of service accounts that + GitRepos are allowed to use. + items: + type: string + nullable: true + type: array + allowedTargetNamespaces: + description: 'AllowedTargetNamespaces restricts TargetNamespace to the + given + + namespaces. If AllowedTargetNamespaces is set, TargetNamespace must + + be set.' + items: + type: string + nullable: true + type: array + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + defaultClientSecretName: + description: DefaultClientSecretName overrides the GitRepo's default + client secret. + nullable: true + type: string + defaultServiceAccount: + description: DefaultServiceAccount overrides the GitRepo's default service + account. + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: gitrepos.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: GitRepo + listKind: GitRepoList + plural: gitrepos + singular: gitrepo + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.repo + name: Repo + type: string + - jsonPath: .status.commit + name: Commit + type: string + - jsonPath: .status.display.readyBundleDeployments + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepo describes a git repository that is watched by Fleet. + + The resource contains the necessary information to deploy the repo, or + parts + + of it, to target clusters.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + branch: + description: Branch The git branch to follow. + nullable: true + type: string + caBundle: + description: CABundle is a PEM encoded CA bundle which will be used + to validate the repo's certificate. + format: byte + nullable: true + type: string + clientSecretName: + description: 'ClientSecretName is the name of the client secret + to be used to connect to the repo + + It is expected the secret be of type "kubernetes.io/basic-auth" + or "kubernetes.io/ssh-auth".' + nullable: true + type: string + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + deleteNamespace: + description: DeleteNamespace specifies if the namespace created + must be deleted after deleting the GitRepo. + type: boolean + disablePolling: + description: Disables git polling. When enabled only webhooks will + be used. + type: boolean + forceSyncGeneration: + description: Increment this number to force a redeployment of contents + from Git. + format: int64 + type: integer + helmRepoURLRegex: + description: 'HelmRepoURLRegex Helm credentials will be used if + the helm repo matches this regex + + Credentials will always be used if this is empty or not provided.' + nullable: true + type: string + helmSecretName: + description: HelmSecretName contains the auth secret for a private + Helm repository. + nullable: true + type: string + helmSecretNameForPaths: + description: HelmSecretNameForPaths contains the auth secret for + private Helm repository for each path. + nullable: true + type: string + imageScanCommit: + description: Commit specifies how to commit to the git repo when + a new image is scanned and written back to git repo. + properties: + authorEmail: + description: AuthorEmail gives the email to provide when making + a commit + nullable: true + type: string + authorName: + description: AuthorName gives the name to provide when making + a commit + nullable: true + type: string + messageTemplate: + description: 'MessageTemplate provides a template for the commit + message, + + into which will be interpolated the details of the change + made.' + nullable: true + type: string + type: object + imageScanInterval: + description: ImageScanInterval is the interval of syncing scanned + images and writing back to git repo. + type: string + insecureSkipTLSVerify: + description: InsecureSkipTLSverify will use insecure HTTPS to clone + the repo. + type: boolean + keepResources: + description: KeepResources specifies if the resources created must + be kept after deleting the GitRepo. + type: boolean + ociRegistry: + description: OCIRegistry specifies the OCI registry related parameters + properties: + authSecretName: + description: AuthSecretName contains the auth secret where the + OCI regristry credentials are stored. + nullable: true + type: string + basicHTTP: + description: BasicHTTP uses HTTP connections to the OCI registry + when enabled. + nullable: true + type: boolean + insecureSkipTLS: + description: InsecureSkipTLS allows connections to OCI registry + without certs when enabled. + nullable: true + type: boolean + reference: + description: Reference of the OCI Registry + type: string + type: object + paths: + description: 'Paths is the directories relative to the git repo + root that contain resources to be applied. + + Path globbing is supported, for example ["charts/*"] will match + all folders as a subdirectory of charts/ + + If empty, "/" is the default.' + items: + type: string + nullable: true + type: array + paused: + description: 'Paused, when true, causes changes in Git not to be + propagated down to the clusters but instead to mark + + resources as OutOfSync.' + type: boolean + pollingInterval: + description: PollingInterval is how often to check git for new updates. + nullable: true + type: string + repo: + description: Repo is a URL to a git repo to clone and index. + nullable: true + type: string + revision: + description: Revision A specific commit or tag to operate on. + nullable: true + type: string + serviceAccount: + description: ServiceAccount used in the downstream cluster for deployment. + nullable: true + type: string + targetNamespace: + description: 'Ensure that all resources are created in this namespace + + Any cluster scoped resource will be rejected if this is set + + Additionally this namespace will be created on demand.' + nullable: true + type: string + targets: + description: Targets is a list of targets this repo will deploy + to. + items: + description: GitTarget is a cluster or cluster group to deploy + to. + properties: + clusterGroup: + description: ClusterGroup is the name of a cluster group in + the same namespace as the clusters. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a label selector to select + cluster groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + clusterName: + description: ClusterName is the name of a cluster. + nullable: true + type: string + clusterSelector: + description: ClusterSelector is a label selector to select + clusters. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: Name is the name of this target. + nullable: true + type: string + type: object + type: array + type: object + status: + properties: + commit: + description: Commit is the Git commit hash from the last git job + run. + type: string + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the GitRepo.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + type: array + desiredReadyClusters: + description: "DesiredReadyClusters\tis the number of clusters that\ + \ should be ready for bundles of this GitRepo." + type: integer + display: + description: Display contains a human readable summary of the status. + properties: + error: + description: Error is true if a message is present. + type: boolean + message: + description: Message contains the relevant message from the + deployment conditions. + type: string + readyBundleDeployments: + description: 'ReadyBundleDeployments is a string in the form + "%d/%d", that describes the + + number of ready bundledeployments over the total number of + bundledeployments.' + type: string + state: + description: 'State is the state of the GitRepo, e.g. "GitUpdating" + or the maximal + + BundleState according to StateRank.' + type: string + type: object + gitJobStatus: + description: GitJobStatus is the status of the last Git job run, + e.g. "Current" if there was no error. + type: string + lastPollingTriggered: + description: LastPollingTime is the last time the polling check + was triggered + format: date-time + type: string + lastSyncedImageScanTime: + description: LastSyncedImageScanTime is the time of the last image + scan. + format: date-time + type: string + observedGeneration: + description: 'ObservedGeneration is the current generation of the + resource in the cluster. It is copied from k8s + + metadata.Generation. The value is incremented for all changes, + except for changes to .metadata or .status.' + format: int64 + type: integer + readyClusters: + description: 'ReadyClusters is the lowest number of clusters that + are ready over + + all the bundles of this GitRepo.' + type: integer + resourceCounts: + description: ResourceCounts contains the number of resources in + each state over all bundles. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + resourceErrors: + description: ResourceErrors is a sorted list of errors from the + resources. + items: + type: string + type: array + resources: + description: Resources contains metadata about the resources of + each bundle. + items: + description: GitRepoResource contains metadata about the resources + of a bundle. + properties: + apiVersion: + description: APIVersion is the API version of the resource. + nullable: true + type: string + error: + description: Error is true if any Error in the PerClusterState + is true. + type: boolean + id: + description: ID is the name of the resource, e.g. "namespace1/my-config" + or "backingimagemanagers.storage.io". + nullable: true + type: string + incompleteState: + description: 'IncompleteState is true if a bundle summary + has 10 or more non-ready + + resources or a non-ready resource has more 10 or more non-ready + or + + modified states.' + type: boolean + kind: + description: Kind is the k8s kind of the resource. + nullable: true + type: string + message: + description: Message is the first message from the PerClusterStates. + nullable: true + type: string + name: + description: Name of the resource. + nullable: true + type: string + namespace: + description: Namespace of the resource. + nullable: true + type: string + perClusterState: + description: PerClusterState is a list of states for each + cluster. Derived from the summaries non-ready resources. + items: + description: ResourcePerClusterState is generated for each + non-ready resource of the bundles. + properties: + clusterId: + description: ClusterID is the id of the cluster. + nullable: true + type: string + error: + description: Error is true if the resource is in an + error state, copied from the bundle's summary for + non-ready resources. + type: boolean + message: + description: Message combines the messages from the + bundle's summary. Messages are joined with the delimiter + ';'. + nullable: true + type: string + patch: + description: Patch for modified resources. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + state: + description: State is the state of the resource. + nullable: true + type: string + transitioning: + description: 'Transitioning is true if the resource + is in a transitioning state, + + copied from the bundle''s summary for non-ready resources.' + type: boolean + type: object + nullable: true + type: array + state: + description: State is the state of the resource, e.g. "Unknown", + "WaitApplied", "ErrApplied" or "Ready". + type: string + transitioning: + description: Transitioning is true if any Transitioning in + the PerClusterState is true. + type: boolean + type: + description: Type is the type of the resource, e.g. "apiextensions.k8s.io.customresourcedefinition" + or "configmap". + type: string + type: object + type: array + summary: + description: Summary contains the number of bundle deployments in + each state and a list of non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + type: string + type: array + state: + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + updateGeneration: + description: Update generation is the force update generation if + spec.forceSyncGeneration is set + format: int64 + type: integer + webhookCommit: + description: WebhookCommit is the latest Git commit hash received + from a webhook + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: imagescans.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ImageScan + listKind: ImageScanList + plural: imagescans + singular: imagescan + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.image + name: Repository + type: string + - jsonPath: .status.latestTag + name: Latest + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: API is taken from https://github.com/fluxcd/image-reflector-controller + properties: + gitrepoName: + description: GitRepo reference name + nullable: true + type: string + image: + description: Image is the name of the image repository + nullable: true + type: string + interval: + description: 'Interval is the length of time to wait between + + scans of the image repository.' + nullable: true + type: string + policy: + description: 'Policy gives the particulars of the policy to be followed + in + + selecting the most recent image' + properties: + alphabetical: + description: Alphabetical set of rules to use for alphabetical + ordering of the tags. + nullable: true + properties: + order: + description: 'Order specifies the sorting order of the tags. + Given the letters of the + + alphabet as tags, ascending order would select Z, and + descending order + + would select A.' + nullable: true + type: string + type: object + semver: + description: 'SemVer gives a semantic version range to check + against the tags + + available.' + nullable: true + properties: + range: + description: 'Range gives a semver range for the image tag; + the highest + + version within the range that''s a tag yields the latest + image.' + nullable: true + type: string + type: object + type: object + secretRef: + description: 'SecretRef can be given the name of a secret containing + + credentials to use for the image registry. The secret should be + + created with `kubectl create secret docker-registry`, or the + + equivalent.' + nullable: true + properties: + name: + default: '' + description: 'Name of the referent. + + This field is effectively required, but due to backwards compatibility + is + + allowed to be empty. Instances of this type with an empty + value here are + + almost certainly wrong. + + TODO: Add other useful fields. apiVersion, kind, uid? + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Drop `kubebuilder:default` when controller-gen doesn''t + need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.' + type: string + type: object + x-kubernetes-map-type: atomic + suspend: + description: 'This flag tells the controller to suspend subsequent + image scans. + + It does not apply to already started scans. Defaults to false.' + type: boolean + tagName: + description: TagName is the tag ref that needs to be put in manifest + to replace fields + nullable: true + type: string + type: object + status: + properties: + canonicalImageName: + description: 'CanonicalName is the name of the image repository + with all the + + implied bits made explicit; e.g., `docker.io/library/alpine` + + rather than `alpine`.' + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + type: string + lastUpdateTime: + description: The last time this condition was updated. + type: string + message: + description: Human-readable message indicating details about + last transition + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of cluster condition. + type: string + required: + - status + - type + type: object + type: array + lastScanTime: + description: LastScanTime is the last time image was scanned + format: date-time + type: string + latestDigest: + description: LatestDigest is the digest of latest tag + type: string + latestImage: + description: 'LatestImage gives the first in the list of images + scanned by + + the image repository, when filtered and ordered according to + + the policy.' + type: string + latestTag: + description: Latest tag is the latest tag filtered by the policy + type: string + observedGeneration: + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/fleet-crd/104.1.5+up0.10.9/values.yaml b/charts/fleet-crd/104.1.5+up0.10.9/values.yaml new file mode 100644 index 0000000000..d41d3a2444 --- /dev/null +++ b/charts/fleet-crd/104.1.5+up0.10.9/values.yaml @@ -0,0 +1 @@ +# This file is intentionally empty diff --git a/index.yaml b/index.yaml index 8366d95506..6bef74dbf1 100755 --- a/index.yaml +++ b/index.yaml @@ -2727,6 +2727,23 @@ entries: urls: - assets/fleet-crd/fleet-crd-105.0.0+up0.11.0.tgz version: 105.0.0+up0.11.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd + apiVersion: v2 + appVersion: 0.10.9 + created: "2025-02-25T21:44:24.58122284-03:00" + description: Fleet Manager CustomResourceDefinitions + digest: edb134bd5d36eb79e5228978de5c7689a57653a149e219bb289d7af40a882f88 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-crd + urls: + - assets/fleet-crd/fleet-crd-104.1.5+up0.10.9.tgz + version: 104.1.5+up0.10.9 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index 7a1c4f2b64..dd16d16ef4 100644 --- a/release.yaml +++ b/release.yaml @@ -6,3 +6,5 @@ rancher-logging-crd: - 104.2.0+up4.8.0 fleet: - 104.1.5+up0.10.9 +fleet-crd: + - 104.1.5+up0.10.9 From cdd36aefa5b92eb3e9270a7af122c243eeaac25c Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:45:31 -0300 Subject: [PATCH 07/19] fp: fleet-agent 104.1.5+up0.10.9 --- .../fleet-agent-104.1.5+up0.10.9.tgz | Bin 0 -> 3488 bytes .../fleet-agent/104.1.5+up0.10.9/Chart.yaml | 15 +++ charts/fleet-agent/104.1.5+up0.10.9/README.md | 8 ++ .../104.1.5+up0.10.9/templates/_helpers.tpl | 22 ++++ .../104.1.5+up0.10.9/templates/configmap.yaml | 16 +++ .../templates/deployment.yaml | 108 ++++++++++++++++++ .../templates/network_policy_allow_all.yaml | 15 +++ .../patch_default_serviceaccount.yaml | 28 +++++ .../104.1.5+up0.10.9/templates/rbac.yaml | 28 +++++ .../104.1.5+up0.10.9/templates/secret.yaml | 10 ++ .../104.1.5+up0.10.9/templates/service.yaml | 9 ++ .../templates/serviceaccount.yaml | 4 + .../104.1.5+up0.10.9/templates/validate.yaml | 11 ++ .../fleet-agent/104.1.5+up0.10.9/values.yaml | 70 ++++++++++++ index.yaml | 19 +++ release.yaml | 2 + 16 files changed, 365 insertions(+) create mode 100644 assets/fleet-agent/fleet-agent-104.1.5+up0.10.9.tgz create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/Chart.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/README.md create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/_helpers.tpl create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/configmap.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/deployment.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/network_policy_allow_all.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/patch_default_serviceaccount.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/rbac.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/secret.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/service.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/serviceaccount.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/templates/validate.yaml create mode 100644 charts/fleet-agent/104.1.5+up0.10.9/values.yaml diff --git a/assets/fleet-agent/fleet-agent-104.1.5+up0.10.9.tgz b/assets/fleet-agent/fleet-agent-104.1.5+up0.10.9.tgz new file mode 100644 index 0000000000000000000000000000000000000000..470454e910e7cdef56a99c0d1a13ddbd3e7fca41 GIT binary patch literal 3488 zcmV;R4PWvfiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI|CbK|&?p3nRhU38^pXV0W1`DJ!lcUR@}coXM5&Xhfy)YWEd z0+B5VF$i!0P+mo=|9*uRQY1xNGZT-U&8k1xBG3RDK;uDogT#a(>WEB{>w~iykvd+G zoV|Ed8Vm-5lcOX1e=rzy{|^TzM=yp)uaA$9hJ(YS;fulW@bLBU1q>cnjD9MWCi2DL zpKhz$xnBtp&V?qL3a*j>ASIeGF^yBAHN%*SgA}Ev6N*^^lJInfvX^Z}vkZ9xp_V0v zy)3uo7^68#MFmgb<$wPd4CCQ9@gN$&f5I?69Gagmdy8_Cqbf*>3An__>8)BQjd^bo zp%P$}mp}LB7ASM7RaA5B-%*~4JJrkWJKh$46o2bwAH~DL{jy7BNEAk<4+%8ZTR;l> z=c>l}C?G{qKMqWl-v$|~RMJA*wC_w&cuP2$qJ&dZ1w`z5_-aq7-U%z{FIj11n9e=F?zPy1KK&!BISnkN=}Ayj0Jvvpae`YU2N# zuoBhdZvb2H|7dtT?Bf5)==Bc&pMq#krf87gAYtf1aNk0rL2-$NP*e-KsNw&h2@(LB zOcR)UWC-@){S2Wf$Bd@%{q-e?3FsNZ|CVEv95pI9eRly0o+a z&()Hq+NH;VmGb@N&0CRS0%5agJwLt$kT|7EOKXj|NeZ;MwN{1Yy(D8~%8Hm}=AUj9 zE(^iiH3-fYv1wodL?UpZO*zbfOtc@v2}!k?M`mkIf;|A32$8^;$hCCT48z5nN>8d5 zjT}meloE|_NA)altQT*Z%2%5%-Qss>#d0E>XIF#Gyjt5keKnFAJgwH0oS{;%5G6_VG7|0W!Ij9+>w_&-h|Ekgo$}lP-rgQu<75p(xAhV?=2a$A zbx&XIL5evw_MjPU_FL59em;p^S`}dlU03d=} zTqL%#IAx;D?3iXK0F7-~J%np{ zkY^jegXQwz>)=}q$HKG7MO zUNMy4_nZvvy=WapYt;1II=iFCqkjB%23$^xM+pXP!T-U@;i!ZEC!<~b|7qx!@+^Tf zJKt|f(T+(EgB&%gfLkywfLhF zyu-O#?XT-G$g?h4`PpF-E|*oQDsi<6tAb%O`Vc12;BMxuR$;HiQzFM?if4j(cj3Y~ zuXDm0qTI{VZth;TR@;qUuAazX?qB~6m+%n)V9WYHe!aQ=kB2+^?@6e+-bqoYgZVIU zm~f+s#!1O;P;Y?&RbXo4IVk5e?TOHsEbm)e<<5ytXO}+a=Cq#_MYl|)j=oGo1VgM< zyte{?N-{&nsXlXoGn92fW7N#owM4#qSEp~!Z{D4ro!2?u3j0oqJZU5XOenJK8Yk^k zpZ<>MSyIa$*CD)`*TvuA<#Ji~_W?sZhSh2fDzN{?|ZhibRo3+)RlW&XG0JmrMAm6dK#L*;qoe%Gx(* zq$x~GN%g`^HU6xdvtq+p?0hhS_u6eW#}d(0WHfW zTT#`OZB~(zf{ZDnnxbl}MkYnkPKn_3^0M8l=6;v%v~~kfs*(8^W!-7!m(omtxnO0E zZ$-&%=$x;}i-M3?H5(YU|#PVv8#;yblhoBG;ObbB!T zhE`_^ZZxcwbeg7Sf%F+mJyW+nrivRUCTn?G$0&^<*$}*Q(6I!{Q&wj8MTBDb1vupy zat+_WW*D#`SL^)Ea)K8BM{>56`4{N6&=R?ItaXpXtx(!02;GeDZIijld#}j8DK7MV z%kJLQAWMF#t^_w4-qCBD^}Rxwz;G}aw9|htMOpZah9n`S_Wqd>Y?=V?)5TL+J74oz;mRm(cH$k<}LS z*XQ`d{pGS=5!Fioapw&HqqddHR*0-~AWQjFqVh+nHU+y0PG8?I3iPQLu8jSC?tVV^ zi2rL*|N1YCgI=3Ge#ati$NF>(*s}gdgX8Y{AB}eF|7obZ{>NtK*K7WdV%)RRt$I|i z>5u@R^ANcTZ(8l_bMW_9#P!8Y8!u!vt)iD$^7( z!)zbMI1v&fo199t?bf)_jv6%sKp$s#Ni+(f9m*Ndo7XXpK0n+Ih#dq+-j4`B#gFJ7 ziuxBU8Xg5$m*PG?O^kgJwqlT&4yjCi>Jk$c6DU&f}bMs>SE+spaX z)@-c=+f3O$#s`EQlk`?hCYLm)I)UMC-~6|v{{3H$N&0x-0c?H$=lHPu{?F*}aJTGa>0eEI@4d0B0`FbS)S@d?@5!-;G*l4W^VVnK8~IAO*FatGCo&e*MyZ zZ=ggeQ*0&9QH~o45pcn;J>&P+mp&`}I&AZrgsyq>X`R*mAC|28 z{D2?}OW@7dSgL{1uFvMX_Zw7z?4z7!M4OM!CjcA{T5KqJ3G%k8oxbyaQ%OF#J5e` z*8gzO-Tz-7?)<-}p@-Ig@Bcmx zEthlx9AVSr|ESa2q zUK3uztzx`#av;y#H%>kM*{oypoAT%C&88%M9csKz(sX8;LWivks=$AoY(GqMS843q z@8WZzG%$LLYypHR(R7BwX4}uLn>+gQ)WrYm^V2tP&*MCMV9Z`DV^G!=r3(ITfR=D!FS=1^FJNd~udptt>brS75^o9@2f>Xd zJR?l-s^wCdqNPH2LytrEiI7f$$lbl5F>BqX62o^wLM|jihMG{OKq0~p50&}xtLjP$ zcgGi(E^014T8uKV%|>I~*&j@qw= 1.18.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: fleet-agent +apiVersion: v2 +appVersion: 0.10.9 +description: Fleet Manager Agent - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-agent +version: 104.1.5+up0.10.9 diff --git a/charts/fleet-agent/104.1.5+up0.10.9/README.md b/charts/fleet-agent/104.1.5+up0.10.9/README.md new file mode 100644 index 0000000000..2c5724dcef --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/README.md @@ -0,0 +1,8 @@ +## Fleet Agent Helm Chart + +Every Fleet-managed downstream cluster will run an agent that communicates back to the Fleet controller. This agent is just another set of Kubernetes controllers running in the downstream cluster. + +Standalone Fleet users use this chart for agent-initiated registration. For more details see [agent-initiated registration](https://fleet.rancher.io/cluster-registration#agent-initiated). +Fleet in Rancher does not use this chart, but creates the agent deployments programmatically. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/_helpers.tpl b/charts/fleet-agent/104.1.5+up0.10.9/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/configmap.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/configmap.yaml new file mode 100644 index 0000000000..5b052e8745 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/configmap.yaml @@ -0,0 +1,16 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fleet-agent +data: + config: |- + { + {{ if .Values.labels }} + "labels":{{toJson .Values.labels}}, + {{ end }} + "clientID":"{{.Values.clientID}}", + "agentTLSMode": "{{.Values.agentTLSMode}}" + {{ if .Values.garbageCollectionInterval }} + "garbageCollectionInterval": "{{.Values.garbageCollectionInterval}}" + {{ end }} + } diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/deployment.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/deployment.yaml new file mode 100644 index 0000000000..a13d85abee --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: fleet-agent +spec: + serviceName: fleet-agent + selector: + matchLabels: + app: fleet-agent + template: + metadata: + labels: + app: fleet-agent + spec: + initContainers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent-register + command: + - fleetagent + - register + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent + command: + - fleetagent + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /.kube + name: kube + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent-clusterstatus + command: + - fleetagent + - clusterstatus + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumes: + - name: kube + emptyDir: {} + serviceAccountName: fleet-agent + {{- if .Values.fleetAgent.hostNetwork }} + hostNetwork: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.fleetAgent.nodeSelector }} +{{ toYaml .Values.fleetAgent.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.fleetAgent.tolerations }} +{{ toYaml .Values.fleetAgent.tolerations | indent 8 }} +{{- end }} +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/network_policy_allow_all.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/network_policy_allow_all.yaml new file mode 100644 index 0000000000..a72109a062 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/network_policy_allow_all.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ .Values.internal.systemNamespace }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/patch_default_serviceaccount.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/patch_default_serviceaccount.yaml new file mode 100644 index 0000000000..aad4eea415 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/patch_default_serviceaccount.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-fleet-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + spec: + serviceAccountName: fleet-agent + restartPolicy: Never + containers: + - name: sa + image: "{{ template "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", {{ .Values.internal.systemNamespace }}] + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.kubectl.nodeSelector }} +{{ toYaml .Values.kubectl.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.kubectl.tolerations }} +{{ toYaml .Values.kubectl.tolerations | indent 8 }} +{{- end }} + backoffLimit: 1 diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/rbac.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/rbac.yaml new file mode 100644 index 0000000000..1a7e8d8841 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/rbac.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-agent-system-fleet-agent-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - "*" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-agent-system-fleet-agent-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-agent-system-fleet-agent-role +subjects: +- kind: ServiceAccount + name: fleet-agent + namespace: {{.Release.Namespace}} diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/secret.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/secret.yaml new file mode 100644 index 0000000000..4715882047 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}" + clusterNamespace: "{{b64enc .Values.clusterNamespace}}" + token: "{{b64enc .Values.token}}" + apiServerURL: "{{b64enc .Values.apiServerURL}}" + apiServerCA: "{{b64enc .Values.apiServerCA}}" +kind: Secret +metadata: + name: fleet-agent-bootstrap diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/service.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/service.yaml new file mode 100644 index 0000000000..36d35588f3 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: fleet-agent +spec: + type: ClusterIP + clusterIP: None + selector: + app: fleet-agent diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/serviceaccount.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/serviceaccount.yaml new file mode 100644 index 0000000000..73e27f0be9 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-agent diff --git a/charts/fleet-agent/104.1.5+up0.10.9/templates/validate.yaml b/charts/fleet-agent/104.1.5+up0.10.9/templates/validate.yaml new file mode 100644 index 0000000000..d53ff1c508 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/templates/validate.yaml @@ -0,0 +1,11 @@ +{{if ne .Release.Namespace .Values.internal.systemNamespace }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.systemNamespace) }} +{{end}} + +{{if ne .Release.Name .Values.internal.managedReleaseName }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.managedReleaseName) }} +{{end}} + +{{if not .Values.apiServerURL }} +{{ fail "apiServerURL is required to be set, and most likely also apiServerCA" }} +{{end}} diff --git a/charts/fleet-agent/104.1.5+up0.10.9/values.yaml b/charts/fleet-agent/104.1.5+up0.10.9/values.yaml new file mode 100644 index 0000000000..089fca8277 --- /dev/null +++ b/charts/fleet-agent/104.1.5+up0.10.9/values.yaml @@ -0,0 +1,70 @@ +image: + os: "windows,linux" + repository: rancher/fleet-agent + tag: v0.10.9 + +# The public URL of the Kubernetes API server running the Fleet Manager must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# The the pem encoded value of the CA of the Kubernetes API server running the Fleet Manager. +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# The cluster registration value +token: "" + +# Labels to add to the cluster upon registration only. They are not added after the fact. +# labels: +# foo: bar + +# The client ID of the cluster to associate with +clientID: "" + +# The namespace of the cluster we are register with +clusterNamespace: "" + +# The namespace containing the clusters registration secrets +systemRegistrationNamespace: cattle-fleet-clusters-system + +# Please do not change the below setting unless you really know what you are doing +internal: + systemNamespace: cattle-fleet-system + managedReleaseName: fleet-agent + +# The nodeSelector and tolerations for the agent deployment +fleetAgent: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: [] + ## HostNetwork setting for the agent deployment. + ## When set allows for provisioning of network related bundles (CNI configuration) in a cluster without CNI. + hostNetwork: false +kubectl: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: "Equal" + value: "true" + effect: NoSchedule + +global: + cattle: + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.29.0 + +debug: false +debugLevel: 0 diff --git a/index.yaml b/index.yaml index 6bef74dbf1..833ac271fd 100755 --- a/index.yaml +++ b/index.yaml @@ -2014,6 +2014,25 @@ entries: urls: - assets/fleet-agent/fleet-agent-105.0.0+up0.11.0.tgz version: 105.0.0+up0.11.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.18.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: fleet-agent + apiVersion: v2 + appVersion: 0.10.9 + created: "2025-02-25T21:45:13.557652278-03:00" + description: Fleet Manager Agent - GitOps at Scale + digest: c45b998311a1aaa76c25f8bf007aea69cf21cc3965e9340db59cc1a6beaf99f2 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-agent + urls: + - assets/fleet-agent/fleet-agent-104.1.5+up0.10.9.tgz + version: 104.1.5+up0.10.9 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index dd16d16ef4..a8f4962ab4 100644 --- a/release.yaml +++ b/release.yaml @@ -8,3 +8,5 @@ fleet: - 104.1.5+up0.10.9 fleet-crd: - 104.1.5+up0.10.9 +fleet-agent: + - 104.1.5+up0.10.9 From 930d94b41c50fc16e759761e09826bf1eaa340da Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:46:18 -0300 Subject: [PATCH 08/19] fp: rancher-monitoring 104.1.4+up57.0.3 --- .../rancher-monitoring-104.1.4+up57.0.3.tgz | Bin 0 -> 479438 bytes .../104.1.4+up57.0.3/.editorconfig | 5 + .../104.1.4+up57.0.3/.helmignore | 29 + .../104.1.4+up57.0.3/CHANGELOG.md | 47 + .../104.1.4+up57.0.3/CONTRIBUTING.md | 12 + .../104.1.4+up57.0.3/Chart.yaml | 126 + .../104.1.4+up57.0.3/README.md | 1080 ++++ .../104.1.4+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../104.1.4+up57.0.3/charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../104.1.4+up57.0.3/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 24 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 367 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../104.1.4+up57.0.3/templates/NOTES.txt | 4 + .../104.1.4+up57.0.3/templates/_helpers.tpl | 467 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../ingress-nginx/network-policy.yaml | 19 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../104.1.4+up57.0.3/values.yaml | 5433 +++++++++++++++++ index.yaml | 130 + release.yaml | 2 + 573 files changed, 104986 insertions(+) create mode 100644 assets/rancher-monitoring/rancher-monitoring-104.1.4+up57.0.3.tgz create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.4+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring/rancher-monitoring-104.1.4+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-104.1.4+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..88dbaf67497893a96e9dd0e7d53913fa532f3cc1 GIT binary patch literal 479438 zcmV(=K-s?^iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%fiJ+{-TA}Lvk&F-9g6w6LD?T3zJr_Y&Q zJqlO_l8CB+3V@Q#M7eAJ#{GJK+WeBcu<m7s;N)W>(nIb?UbjE9u5hy^Y7R3}Tm8#@)G+Iz-p7C;2 z30@MlB(+R3UY0eZYBgGtqD(RkL`HDmR8k3oOLQXm4_k$+WUl`VUlXGHf&N$%VB zR?A6M@RdwBOSqh(E6Q?yCHt#VSrvGdn1y=b2hv*^*5Hx|NjaOMd;j(onxuPQq~m0a z{(>gy{zU)tUT;$kUsgCHQ)E7dla?!~NZH%2BBG>9!aNGf>bJX_W>X1%Npm8T#if{{ zriV1+f^exfH*NNCbUEpT;0bKSPINDwjMq<8kb+=I?1FZ8cuk`pYa*3QvZ|h< zhX>gc>IF))>W%YFLqH&MMR=FFM6xhR3D&FEt6sNkv#rSVoDQ?j$JVET;;pECilohEu}utRVRg zvgg)*-$6)_ic3AR)fCNXL8hZoIx;7HoA-X*)^E4-y&HeizTa-!H}3k{P2Y0Q z`?q}Kj^AR#*X+0DGuP6ZR{0J6v}@-?W`b4?)ed9}eNSc=DAO2`DTy@KAsA6aq7pMY z*Bsz3`o^$5D93U+<5=W!R|_R9*HVs!ghX8si85Z!C?h$#qH2k9p4BB`Mu39`W+)OV zjw}a_2$n0P_zprwt|5r9JW)I$EDr}{`OAQDG@S%vsHOg@mL&MNgU~B0FQq$64jj_E z$ze3pD`=M+OG%U*6@0P`||9Xm;TLw5Aao?KCGUH19yZBHMdni>p#h>pk{*M&l6+tHkh ze%g}m4AJW)m554&(SIGjc%IC;D6vu`2coSj9FeTRf}l$*sJ>(2BAO9YX>y(;$`nT{ zUJIm1Sru3jneOZi(W$}0Kxu`Za5krlTA1UsGaSZWV4OVy#&C!PnZJ{saijzxBYg{O z5MYMGo$lA^4nmfvm?9-=O-h?uQ_xWk(UT=+3xX7^;ao2R%G@V{yjJbiB@sg3+g!lM z=Jy=+(4w74`#Bx;+?Byrrt36zsl3bplxAwC=<)dR7#98XuTRlQEti^a3w4Sc*4YkV z6s~GcE|bf>^zo>|*#%yZzAN2-s%p8c1b?d+yN{)gmV1s1sn^@8B2HlVlkwicmuQG` zi`$ALNwQiHm>$AXFXqJ~iHzQJZwH}3a|Y<2`CgOKGyhpZ2~&1Q&D5;|y1o7S4`rs) zvkO{%M+BX(qT*VTGa@c&Mh>%#*GxSTB-hs*E@V)#vNIY_COfv-bDrU1$)yUaX=(S2 zXfBfrHz<&R1usZ?;n_UUJ)5~z1!g4BaU-O7K}2H>Kqx_EJ|~%)qQOi4^exG1Wve0- zXvv?xrBca3w~1rOH?|2_GKcKYJ@D{S3C)*H(cmwzRR)qFR&Ji1z2A%X-fe{0eac|# z|KfID|A{uXv$@d*JA`w8`gXZpXY*E@+B>2AvK@*)?!j9bNK3yhy6)Yf>)!rW=sMXJ zUH3mNbOm94D^z{hhpLrT8AP*b@MnJ${`?kOzhm_;H_v!)|I5A!6Qkl;Gv9~X9n*{m zbwtG!jfl!d%}E(aSp-#{X)(I#Hu*fQz%Hl%T#!|#IJqFJph||H#6U6}luGiPvij|- zD%b!7OwKp-l1Qh{LSXWf`lJ9OC9yBp$RjD^$yNSM_GuEE5ETb;bk(_gy) zH!Q*0cDLEND=scbW5(B9>&3qOO&MBmrrT}<$1O6lFB9pkf$u^$ZzL~UUh_Aw3u1;d z*9*>#9u^m;bwL88VVs>3qq2E(`dm)Y-v=YR0HX}&msD~w_=m@bOCn}aX+f0#oFH7$ zZv?L^C<>|z;(gEKL^S5Ctq*-%|kfvk8@`_Wx!-?JF#!3qt${82W@r<;# zN035)q-+taxLFZb`|s!H7S?%%)$;tjSr>)Hf+(oyJA_IbMxu+SsI!Bg`Zz1WXXLk9 z%^l|%prKyu%PB&J$PuM!*9k9Zw(38wnqvL6o{__&7r!k*TVsyjQc8a((R)jo_*Wu! zx4Dr@EEU{Hzab~l_5}C{!seDz;N1po+Ms_n@hWh(*+#^AGjS7HJ_Rh%1pb2T z{nqN)uDi>fe)dCQ*9MXESCzW%G_+3TwRhWF^Zl^=52(%kCqVA4^|_xKa(_x3$FGP3 zJIF!srN-lCv|S4276zR6tit z!VG*%HZ_FiYVk_XGn-I0eo^3Yw0JM!>>$;0^f zLLMFlV;W$w^BJ^iNkxt(D&|TK%1?g*r~uRWW>Q;)NCw9J43%R$qY-9BNwZ0 zv`;xF5|mk%09Z^hN^}!d#4EzUs*(v@5vN8>cUn_^H%~jCP2^piB=6kpZtZ6GtJ-|- z-0l83-R*|G*?ZQ~#J&tAACNGH;D1H37*73W+W7d($6v&xv7L`)4T1b(6M_6q4Ohb< zdQ&X~&WYTq%|G-YhC_5(Gi1N`R0=aP=X#x+5-yNtYi2mRK$jFFjQ$6op(W=RJ%OOZ zAv!Mcg6JKoCHy67!#~*`b`YYvhFOR(^@QUE<92~XwuG4Ben5_+jYbqq9J3rzHYcXNAEAqx z7Q95X8O@qL{Lv*E5qA;e&FyK18<3yWt!Bet{``xLgSIEp9B|4O$gtav`EJ;5jSFV zg(T`h`lovyTIUAW?exuUYy7wOo}ySYxjx=7>eT2R@DiNunynOZg=~SPMC>sHlhO+I#=dzXx)m4s_$yCsjDlSMcP>VuS zoiKC36b-uKs=0=Jas$H_c~a>S>Z1>`t&LlkAYss2^@~T+Ya=w8)`~azn-DF5A2Sy` zB5whAjZpa-@TBE~Fg58{ziG6m2(VZEmAkV{!Mx%g(kp}#{@l5kS4FNavB_b5=7K9O@?@#~b;*os9ZRV{fRsBZd z1vw)bXSsaM)!M2rAaCvyoGr=58mCfjEI+%#)tU-UZL+cb-?+G-Y;i`^*DFO@eVy?9 z)fJ0Y3Zfh;YN9v~FEK6fj25ts{dK;s1rysy#?kaM~!CdP>g}7g}`cEd#>cD)Ro@ zaOAq9@#X7=8z$2Y-8hW~yXDx~`st+Zp1#B4zOI9OLVX`!c`?@)+B}7kxOYnOT8lH* zYtITddNfF8ywVMERlW3P0do&c(IEYNU_qP_1w7*UoN7vPt_h3&Pg7rjq~%#%;Te=a zhp5lfaH-!y>b>G&L*S;DJaqTM&NNZoQA*?}34_OULfv5(*ElG{Pv~?#F&8E0#82OZy=t*k1px#DeQI_lFiPi zhG=ec~OIVWB~`u>vsNOPx6T%bNUueuBh3J z^9%Im^tnvY8)?-Cp`ZssEfrFs3Jax$TK&Z~)a{HSk3t6K(#rf=dez)W3L=uJ3d%cY z*0&#J+LO33&B>*&;P|5>DnT|~%U|TSf|oKO1;okm5x-)M$pHc89UdPY5wKWIl98DFj2IXf`!{11YDijM=M1dJwsz6V~U5OWU6J!VX#5*G|cu4o7a5GRifc0 zb6ODn9a$hqL^h|o)l;a6+obKAm?koyE!^PCR2%`$8k(X7+SHW8c5Hbodnh`gMI>tUwc*9=+n%Aa=o5GKxju^H3w?8cJj=tGqX>SZ|(K z+xNKx;k3@^=_L^>zXD*21zqG4UlN415Z34k`5cYYNjm#RMlwN^^OtC;(Lm_(Y?>_-42S~^8m^e-OG7) zL4+%7AiBkEK+cHT3Zkf)q9tiS8h+**xNnSm0}}S?aMX{^5jC-@GB#iklipkMk~C76 zZjoV0B)U)11uA%k3kQ-Cx$7=)=u7URG)*5k>ok08ZZ0$Zlya{Xn-f8pv3zp|4jv^J zM6a+Q%8d?ex8RGaX^>_MOkX?QtF8xO6qXl&HekpjiAI>r$xI^+F#&K%CFmtqByr0% zie_?;!7rx588B-G6i!*}mwV~a6(BqeIQ^xlWH}J7izxQ;sv!U3o-cy)P+Y=0)XdxLI zHs_1qc4X^3Q={kEebvU#hv%WrvKeb}g6L}Im+|_@O*cIRK1d@S{F6#q6FGW# zu>bf`Kzok-;J=2!uaQ&`Dk-z7$%42bkrKas!*KJ}o3Kg)%`E&2Vwg*D!umH#hx%kac>vrfkle_TJ3rzG%+Vcvo~p7c?%jqaq5D`O zEkv;eWX#`1mCyo_=A^VhW>lQe7%j0}q7qj|va+qr#pFGhYol5#R=ZllQ&nNhsSI4= zKUoqOPAQ=7Cs-OuXuBmlo) zOS3&Wxg-UzC|jIcfoTYDLndlT5>*w+yk_9pdgJbF+-;4^=VnK435ao}MN-3Sjozd2 zxpm$IARd`x@A=-nu@7A+j?6P@jSXiwL|?C*bh7IkJcIgrXTXy;_itqMopTeBFmJZs zDV*W_L*Gc&^ErL%&m~xr^Aw#Ccbrp?M+ndOr9DcTdk1!F7vVfdTrG*Pu!sbl#knz% zL?yer52@Kx&4hy|u7TIzmr;dgMGeL`E=kkHEh)su+3Mj?Z*H_VHa^Fwsn4D02Mg|{ zMg8aJpA-9V2^z=(XBVk$lPdC74N&sSFG1Y~VFqEf^Rg}!tqL+gDT*2-h_D=h^!B(0 zNZ9G;fB)w6)teKPpd+F%E#yC%w^v>T>MvYiQA!gVL#>B{!80Z3VnOr`fl;`GX*?!6 zM+=4S7lffSa8*jb1F7!;8izCc9BNn>vFYc*^{GJ#aQlv+oN=K@euAlxAsmGM&;S1a z(C0t@jMA6-!u{o!>F4Ns{S&%Hzx?uj3$|X*o+WO5qH~l!eXEFIxOj8=JR16Q$}TiU z10`@y1)uBPUDqV&WT$7IhsLR7D@@UZcP*a0GwL{~NzXFa4QQB}2)PcQKRtbo5}18Y z%>db8M1|~Y->%ia9~QjlQbRar&OO@hez)%Tx9GyXjvV%2Z79>vQ8?G0nj63eDwtH( zWr;;=7Ur8prWc6~85f~>0@AEZ9zl?m4P&&*z$oEMkm}`G~sR?x5>DAW4$0BkP2WHa;u~}$f^hY_JgH=)aN9-UZjN8y} zqwKZAu)jgeX{5eFHFK}$@oD$F|)MWH1v!5Etg7qlc9A{gQN^L<+Prh#o)+21>9eJ@TZM! z4UG>uFEM5Cyu9n|1goWVHn=$#ez}dkC;_8zh1Jp>0Z{UEmglS@Sqh3G<9rwnOSg_= zQb)e~+Z{O#3EoG%pA$tyNg0u7$*=svKvzM`vC&B74_tR2_w3DxQbZ(r5DE0GPobvW z=95}g#GO7jlPNmZa+mG~j@y~VMZZ9=pP#k#A|1=mad@tW)^En+Pl;KL3!m*eck%vB zoN2ICs%Qld11y8X5w==i>_ioAMh^Y_&lc`k3DY58uZ1w0mg&2(IO$*0b<}ByW~=7m zf%e%ojrFl%T!CiqN>HT;vyK?J5CrEdlbt|1)g-Lf@!aIjh}9n5q5F7K%se)N(g}1B zC9d=tBRE^eleCGr0-`t^#_Q$?BoU{!uG?8l0=$oe>9whNch$YiRk!A*dqy=MH0Ppp zw}Z(F6zz{6k9}v)sRSqd;}eAQTyH#)NE5~-wfuA7oMoc5yi7wS@_rT$ z-c4S)wbX1`O>;7`VB}~>tOhorm8X{uU?AS>1vEiH71@X*e$qj#hi)5@USq`NxYkt3 zzKdj7!m1_Zwe~6~h@pgO^&&WX>)`B}X}b9zpEN@(IL^Pug%P2ym~3{KD{SQ15??rDl)i^E&KSw8pPS&NK0{b2n$-msHh-P5ykRMui(cx> z;DqPqgeY2iJUq<6`5WDP0&R~^?&;awd&wDzj)gQig?(mYq-(4lG|$}y?7+4GAnSkQ zX~WXG(D@tY-^S=){}uh^t4Civ9FOm7ES{houO8^#avo+5*jHbnf$M3oVc1Usw;-49 z@A#x~aIk~2VW(3x89y42gFDMoyj2y8S#T{sakO&_p~9tO@@}=w6%(YK>Xj{8r2Pi@ zV6FD&MzS7Wbz6}5cct+i;{Kl0J-BsmL+2m*exGxk&n)4vK1rJBgIZMHvgg6`fC~N2 z>Cy*8{+27bN5L1dzwVlZc*k?_zS->pazct@Nu8V$0{2`%OeKoB>n4a1`gOAJ4f?Kd zo7k-PihkY|XU{+x^PHB{DQTrXz=<770R@~PW_V6Mw}Z@mGP+30!*D6IhAf*trl=u` zCb3G~HEwzYnV{2a^B~(zy{ZO$*z3v0$=PHiz0i%&V`r0^lYEH_3KC4Pn?4|MFWvhx z9fO++BUh~uLg;C$(G`R()G`>INlNxB4N=Qq&tU}bHdu2t|5~F3Thf~0e#HEgGFuYO zrGESR^@;rvsw+I8NE(G(?t4{8L-M{iH@VS{Y61@4WQa;dIVxX2KQqH{T?M_HoBLa( zs=X=t+tP@*`T{gDBYIGKX`fo6KRn1q=l6b_6?|r*gpPPtdtJf^S2Qj2p}Qvnj>3*B zB94C2(;mIHT{ws`^XXl!(g}d4pvt*%rSxq(x3^5sj596>m+vx3RY)^pr!H`FaddS? zupk1$7oO3AO#QrHgy~10Y{>A5=ZZ9Vvx{c6TATHAn~<;f*;AWPG??=5B$=v~m2X^W zyQ*%ff9o~<{bz6OvLsLYn8$`I@x{@EjfAj9?2q#4H}QL#OVnp)t_5AqFt!YnJzkX` zerv-{rR#JDbq3SqX;zr!SmcP$eju5;C-uR?IaWADFU*Ob^WG5$aXw8Fc3NNr~ zbO(`iSn*uO7b(vj+=wGJnd14p9mmyhz)hm~{_F`Q6@m+rhlY0C1{UOz|1x#Y=g1|F zZdNNXG`f+}w4pCl_c?`4sLmf~a}w3Eeji z!*C^rj=2wDQ*>+2h*PQiaWp2Fa3O0+z@`bQJHe_N*#@2F9^4Et$nknx{72R(h=N3l zJs|=HuXnA+Q!cR?4n+I~Ib=NL2-}`$v;iHD>lcew)2@TKjaeVHM71VjJlJ2Dao+~> zN+Djulzn`nUiQP2V=X7TmujLuwhLqo3y9}W# zGK&ayxRe2>HFJ+dmjnT{uZeR*rWHwuAEFD;hBXM=OXn#xgtPXk8;5{4D|LUHJ+^}!$1!D>d z!zW#iSyT|dq1WX8N>Dt`8;TQaPI0`&O0z$p>Fdi;aWts-E4!e z9$JjD{_`fR?M~p<=o^jn!(egCXKhoBWN+s->wV)wdlWY2l6POZE)sXUn|j}_kl4n^ z8YJe?w^18yX4 z&`kxfTC+ya`2szI&`I0ojU=exi+R~(%X^>?^$(IWU9fN}jDjyr!>O$wn5h;PL;6yO z%t&)T=iNKG;EU(F?G%L4Mo^$X5y#Y#x0NBTQh)68pmw<;fujms34!enAQsbUprkoIL)=pYE6PT3(me%LnLwNsEF?V>L$Z?~<}w zfxB5Pox({NYJn7bzAZ)ByEMrAJ1k9>Sc!IN7pYlo+@%= zhnJR-geqI9euwCi7j;RV6quH;t^C`ZC)ub~P|!T^_gq-pWI%MYj7a%{=jQp&Q-brq z3985|mXX0fLKefPB#tf}L4K@BPh)_#o^35hg$0!$>wqC;iCJD)$)_2yiRRdC^!{OVsXmR)tV1Od zm~kP6UlgB$atXuO_>04zpVM_!VmwCeq`9*^DScC7f_< zyU-LEY#sVuu}MG8=;K4*#8XTIht~C0r;lW4+S+Vmos9x-;opbh#;AlWiwF8R5OS>B zlr>7j1{W(ma)Y?JXF6O?i3JA*oykH$#GH$git$)a1%|5%An}D{U(C9IdkqZT1xe=MO`i^ooT8(nXQ{(i&4E@%clb4;9#i?YjXG zNF8gOZZWoumsJhvAZJP8!Rswh#`^>3LG;auLRp3lxNSKz-GAKSe4isNJCy|QId5RdT`atrz z8laM3CVgg;pf79F{@f(9`P}&UC|?1oEABnDDvd~-g>HYkcc6BRqeYDcW{LoZ;X=`( zq1hoVvB)z4Y$K3eqp>-V1nDc`;X^*+m?z$rfB2A z7+VNph)qbIrX%K%Dqit|FIImgtI#bVY8cFmO2D5c=NnRdHzeA-XpU)73zFDCzMd4d zKXFDjOyYGjq$!G9x_FjG=9H2ym`g6(T06TxxQ|!a_Ne zUD5H0-_n;{5g)S;`u4~T*Y*15co)Fwf(4h7RzarS_G8FJGmADu?On>Hd7eNb%6@Q; zBaQQl-_$^8*4&M1-Zikfz%WC&R!hQ^9f@-<)J#Xm8KP}7wVed#KKSS8l>)dumRG_vqOJQ3ChV&5LcToDmyx?(L zpr%>u4Z*0|@>I~M>e`c$gB%Qx7#LR8+q3RhM3mHUPP-_l0x3br13e9AS$E6jJoFSc z4-qwjKLCq|5>4ntP!87%goGo*Pd5HZS{#j7WLrJZ7xwztpOF6L0mT8IhwBucdbp-4 z;%+CK6%k>yq>E*-3iNsYF{NN|IPN~4Cl?SW8wwZDGqSM@EpbM;s9HF`S7g@A(uQLk z=9fe$D#@Wavw?EIpcz#zlo7Vt)kTywy8V~In-)HGi5{4~s863gnLM66LVut91KqFc zSwXW0O>|@m-))`d7KRL_DJ5#2ar-Iso?{z>^;-dU#OEU9>-zyJ%AGqRDb^Tuu(;gK+BcoFXK)H8cvnHW6##lx+~3 ziMp|d6^*cg5B#S$+h8ueQ?2E)s*7T)Ev{_k-@zug=M~NU85bEbPe#}S{enXjm#hV8 zIX`*+EGIZq@a{_Zq8lo8OVH{%o0|>Kvr0XyukkDO#y(O5nh-r3Sz*(z%>}t0a3;8v zp%0iXk^Nn1Izw<~EtB)!xwen{3(9C&mtjPaHMVJFH(FJU7`X}(6I478!a_`D7(F0; zM68bmRWguLGJ)k%&lGZ0s36OfQ7%cEn}nf`(#H;GV{~748EB(A4K&6{&YAw3Dr4fn z`O4Y|OUh35PalOk?sLbE)%IrU*4P7HFO6N$Bv4k=C9F-+eHR^UlRK-bc+k3@dUWa^ zrt<*7oB_z^^TPQhW|83-kz z$11!u)QrLVq0=Bh5j-=!p|K9^h`H0Y-Snu3ckH%uMy-}pbiLzP=sD-p&u}FspS^na z&HjI`X4Ple#q;sAZ;uWy_<#P_aq;crG+W8nW8Xrvk(?1yNMc(az4)L1x%WRG{`lD! zXS4q}p0DcX|B5hr>Vsj(^953Z7dm9B7E zz`K&>^R)BM3~JnqTrRr+8|))I_oQY&A(?60aYx@iuU=C&dvdXCx0_9iz-YoN2E^$e?&h$rU>Cli zta+Z;d961gTCpRO)gsw6X%?%20WmKz9btu?ebBuX=|3UwMGlTXkcQN~O6B|0OPwnHmNi+`)K8Qax7!e(#$dez#I}6^3-jrFd z2)Or5d`de@p^x0y4Z$~qP8(qD*!~S2I|}EbVqP}W4V1&p6qjV(3|wckRrtVd!%P-r zuF#AuDTA|U2UIgllAB3t4##Yad1@d&=>XgJy6eMOR({s*4Jqhk+1m*i!i+aYcjm#9 zC0g_#nAO!Q_Qt`8aa@aGKXFF9;&u(T8`jICFcxHrW`bWy{ZWLOSe1pii)czx^t9GY z;pln&mXtGIi^bZ*;y}B1rJeIf|uSK zrITzpOhMu943|x=g{7fFqc?Hm(6#i+Dwt3UY!mhEF>hlU*;c&D#(arAZ|AjokJMNF zPv*7^q_)Ep^mVSc{;H~w3C@5A)hpV_zBWeDi0U{Gk{0$1t0%$Zx7PgKEZ&Q4GVr#R zu9E}RaZ*9*T@UtgwsTefVezVQqKY<$`tV&`tx3AOjG% z{Ca+h2nqHZh3*?R{wxRtH;v%`zM@H3H|*gY8+U7y*mc40Y6wPOEdWGcqiMb*SFiQoC%Wit#T0%UQPGV=tcN#j@-;eU zxtFTJi!)Y*)mU5O#{}1iDDYWj`Hj^qEehZRj@L1R`p^3(c)*<%K8u=7(d0mG zg_&LP=|J z?|HB(fo1Z~4G3UlYe^(p5}X(MW_39*zwbEulKgwGIcz`#EO*e|E60radLJa3VLT_~ z@jjXC<=L0n9>J5n$)ow?F(!KlB>Q6a=<)u;FUALt$D`_E5kMIQnM~3JA3onZI7puF zJ^I29&&;72zD)}!dxYCHq`S3i>qzntJ%fDRpkD_w2;>fVVBMF!_ywv+ySa_V zBMorCS~)aXl9V%&LuT+s8LFpn76jiSMBaj~R=gHCBggp>(Oq?jDV!TWtv8yw{0~f; zEDmsr4TR?|U)9unIj6Z-YJ~4ZcL-}h-z)*^Nf&`^Jc6COv8?9ik*YQcr%|HiHa&D1 z4b9074Dq3lN2cK>{(M``0c_(Zh6yAp^SVeWi;=D3j&mDS6+O@#7{M1Ex+T}C&u>y$g1ki`;*N4Q^5{m{Vl)ik@1TCv>r5=X zT40J1!#73Fn?!#D{o*0$u>-Ese+|SyEM>dYcxvT14U z7qt402s&Ryef!l98QjP}LVxyGsit^8|BNa@nVO@)|0RQqG?kax0Hvqqd3^sykG@_l zm`nE-x!dd$pDEVuu&ifB(}7v;Bg`xzw=edi87cTxZ<^`GF#!ZW@GkOFk>elTCAeJ@ z=|^KPDZPIR=||%R+^>*A`e57$Zd+&2G(KKC=vH}h3OW;5tddaPhU_?#bnjt0PH<6C z=9D~!D@2%~Gn+2IF@RFmTF>`#n)7H?@jM`#-qYtAhU!kTlR_K+uh6*->IKtFP_pp< zG7K?sHz93eWu7MV&s_ln`F528 zh_u-pedN|uQh0%W`DNhv|HeVC-q)@LHTgy4jgAcEra1~K&?c)^=TU}x? z%5b(Mz3uJbcuBm>+3O?Q@_L=bpRMPs{%%n9JL^Ff`eB#@8Tb~<4r2WJ-Zm3Awm{)g0THPn7NBZQ4S#@zJ|M8GHAKot3&$NO6leVEm0|PZ$PYzC-wNR$$ z1zjvb5!OU%T0%`kDmYj05+c=-i1G^2&>|XW4{0Pa2%?%U!9-$y+)RUPw#?n;LU|%K;BB9qK|8i-~V`nhprQSHa3%c|PvF-V0J&_E*Dktb=`v9mGTJH-Y zE0U=P?nd{%F|!ruL`6G(ZOIuI4g;c0$bEg7J4iZV-A2sOl1PQis=v3uCvba6o{Q`b z_O+|j9`GG|e8(Q&vB&TGW{$NZ^wN4wX$>P_QijDsFXF4 zqsMz+M05ZE#;3nMZm0PhW!T(CCc4B<<(}+&HPc;O+k&nHpSiG=jet$XZNWO#&t|~h zg5$j2!zb2fB*XebXly`^&|*P?_tYzHt%3f|j`%ORd{OEFm zi)x7{NPrFu43eE%Q9mcK&bA9`@COyB+~U8uL9hY`9Fu4<*x4s1Z@l*0i&bmI0Vkhq zlDqDC;OALYpWn~s@?0$iSE?Yn7rhi$xVSDj&d>2B5qLq)Cu4bSEp2hFS`ra|dT?)> zxs4DmuJB5tpNQaSP77kdf4m9L<@tTe&$C(xErC&fuDO!dblle~LKjQbSrdJ_V2tL-+m z5n6J>DR*QAILOGvQX%mtO$^el{s+ z0^pL5HVZw3^z-%z7lPMS%r-OLWUlEM0L^!G`Ih~sfBN&ieo4ET%C51oZ)YG0w&m|Q z`RV*0e|qreev!7vAmWF(VKlcM1|#ZJ?;-MNde|m;4{kS| zKjs}~Se)(+w$c7}hfW_3oqm7%z2VT&F^Ir5X%m_lT9dimrM^X|5I%Psz6F%FxbtG#0x z?pTHo%QA>NGIQ&vfr9!}DS>E2%F%YNs}!)B!BVU6Airj20| zlI)tK*fwUP33mYft$i`J8>-O_oihEhx`61%nt5}iF=l`>t4+L@qkP41Ni&E(pJ8e8 zQ<%QPG;r34f$}gdkL!)ky%HLRPg&S>lJ4zwvR+PMd^@NW*Wo`7Tkbi-*cS(MG0n~= zLAL8YLvrsL-TCp|87l7#mA@-PrJ)=z?OVm3TtfOUuN^g$cZfU}HoZqgd?4Fy@MwML zCFuTT&Q#~I&~@4Ox1&zIR=_p&L)&Y^zF9{XlatZJm7@EGZWZ&yq*XRgkmsZz<`wuj zu4|-!azTGGQC{$rYSY8Vx;kR74r9#)a&MRL?1EM%i3$;l&h=P{K~tBYlCYR3?epCZ zA%=KjT@k~Vu^eo+Xm@LOzHu54p#F1-Vs}oA|E#enA6Fd6xFAPNrjVkTP@BVzN=ivI{mVt3X}2p;Gz8z@*vH3 zd9liw?6x{s)9T>8S(iIT<&IIgV^n@ujLID=amPyBu@ZMa6ydg5eq*!u*>^8v=jdR_F9?DK}n-`ltEriL88{{WUi zX}@@b{Tmj(o#)e7Lu|fGmuVkX+&%9$TcLvE+&z&AIp`@xrt6_hZQWJD;pPP&thfHEAmvR6s>aZ|1y(<(686g>q{u5bSdarO}}@8|PR<8ZtGQ5kIBxd$z`?B{N9);evnanN3*N&Z$7zu7IJY~_?cF%HPhmAbvTjum z%aIwcU)3MYsag9LPWvJGZuUX{#G?Zn7f-`<>#r_}5Hxp2ATh&P?>pwgyg6xUn*A6& z@oY{=$Rzc?`So!)CDc5r+~t-&KED)n-^Vk3RIVxLyN`4FG|AoP<<8=wZzx2p@}~`E9P>V7}|wnl)GPhrW!rxsKn5z4^^oxR1$s zcLmiN71cT=)lF)u^@^&sswyot^y@>Kw*ij;4WKLk`iD^&c~r6Q3_~5@fb>6}+PZy_ z-DWc=Tbx1KZ~qt~tKass9QsR+#T8|1S(4!pJ>w!HX1qsOE@vEz96d2FuIB!Of1w?b zOTrMHqj0#WL3^~s8V?Ecx%iz1Y~It63>U4-+y$tpyG#61hAiH?N}CwHiszBY3G6Q1 z0QVvo_Y9TPd*$eJ^IW)lL)Ir-lM1uktX5fah9GU7zRsHDYt2LRZUsWNx)GbBQLJM? z;Z5K}|A2IYk|g7W5wX)IL3|3Y94AKon2@(wQRn2V|24XwlgsN$@a$TyLU(UwU|(J|Z*!sT1&B{ve)#8P z>~DCQVhb)4I8(S=HD52!*<7CA*X)tHw!9i=aF;Nt?~MoN>7Q=Jb2j;D@A&^uhX2>6 z1=7$(Zz^hXb660eT#7pHfg)oT^_5-G9`z<{)Ei0Ay`O(Zij-BMS@40gZn!;>3TGGT zye1lM=$-|#)}-LM>8ymdAv*zg(>sFNUxW~Q#0>b zG#T%rv7y%sEM|B?96#8B2yg_sq?$y3OEpWjvSfEMw&OWd0xPmuIa5?4QFv!lTLW~> zrctv4iCb87IZpQ<+Z#ZypcWP?Fl@bkeufxU@EC9EpeiQ=^jNhKTob6QrLr8p^F9T+ z!suH3sp17@iQRgGFx(f;ogE*L@-tFHL zVWk(> z6<35Q3J!4=yi3X4Jbq;KX$lKv2&!MCzu6`766ANlmcaRtQ}hheViz4R7#GC-{fZrH zBq$2a#<*rZMJUmH#in)FKD(3T^Yk8ZCR5u^KO?p+s5;INl=O3(y}g_bH6_}7ku0bPV$;#G(?B!MQWFn4MgPTX zI8C)X?&cg^(wrC{Cw|3T*TyuCmjvfPhG|K3$17f=D=L@qYru?g!vsXeKZM1CXL+js zss#TbrF^7UVTw)!Ia{&priw{U3Zh6-S(N>j^NX`O%LvJdjdAiyowLxLz^J?Yt!#LND+&COxadDw# z(}d3xwIs=mzg^2I=mo=g_@ZIX_5<8Gl=R}(FLsQ&rKCTr9LGCQ7@S0x`7Dkn#u4jPx!(5Xq5~)55{4a?3-}o5Tj>j1l@2j&pFO#xWFtU0v(_9 zEpy*04|Kb)ZM#8V$3;-VZgpSLnR!+Uvjyo6?uan1h%`(FmHs^EM1tp>ruw{A1d+Ff z3?L=&d`>f60+tHMDghY+4bml7n!oTq&wT@TuwENbjzxW907T<+slS3 z?4oeR`uf#CrcHx*m#9?w+B-hc8!CvDT3S(-Pi^VJl=k&hik`rC^h?!Fgg*G3#wL63M?f!(kgg=R+_WiEN#`U*t#r^ zz}kVH^96baLalH4MiNx;#k^GLzGVU)=!5oy^|-`9*}Sa)F`l1ZupGva%jtpH{QHxoE+)XYSKLJyjl-_!iSljEaPBQLd; zo)T0^y0v8<+pE9iQoS^+WmgShX@uc0G(#f|pP?@iESe+|Ia+U7+sst}gJvQxC}$m+ zNEde%DMqg6g)?LMjk2$-=LBi#D?qjF`Rzy{HUcmSW_hA~cb!2tMQt0>1zAne;DW3M z0oJ)2Vu}V&f2?s4mVts|iU#m^SU~1;lBp>gyyPx1#-MxYTa?p0E7O$LtJEQWrYoVm z_xfPy7erpY?46}?BWY=VblpQ&6nsbik;tQ3P`0pBrEFmo$@cTpwvf)7<`oki)`5*N`jc2r1COLoq&|{5>uAi{D7^>aJ^Il@KiFp}^Iw&J+#y2E9#4&g9r@9E^L^6V4QQ8|X;p@*BbH zYKp!%IM{DyENXVRq2wiJr<|+SH3~&wL$|u)LVteVFPha(H-OA#k;8!PtUebar$Wxi&0voGy zta;L`$}3i^^nyO~4tK4k&2WWhw4jQHIEcWRApEU&|M2CB1VJD zD}{Isva#2dQMN#bC&wrgBsX?6*-^1>R;)G7a9}lA@5S09L~IZhC|hXKlduJ4ZxgVe z+p8KfM=l6!ar%?=VY+{NM&AvkhqnST32ZFb%YG=+!k_ie;*p66rmxrY2R`daqP?P{ zUEW@8;MQiC_b|lRv2~?nQ_x{5?0CC(Rr_XdRfgfzjdCIje3=w)%-`OqP23ye`d`7r)TnE7b;}kS&f%=JbKoHO{Dg>L zIdXenYpSP#c&7#M$Uktp4KJ15ruC%Jj8G|SB2LLX6lkqg$CQPl!Ubjd6b+mb*_K%p&55FXFBe)kvYGM#!=xuU<4|tR1eafhI~4Xl3rT* zEGKN$j3Y&7h5~vA0L&06E44y=_5;bR`+$wtaQ-h5s|0k%+ucKpzYR?Ay+!_0ja9^j z_9r%A7m&5PVp@uRmsPv3XB;lk4r(oeyIh7t^o$mYh+sOp%R*yyE9S)OH?7d;uX@)7 z4qX1gNrz>m)n#1C$QPX|O`mI_s3m3`JgL_8)cERI8RG~<3~Q11j@96MW%&j%I z`Q-HZx@dea@Ff>+V{MPf@O695)yP$$<9fhCabP=HYirMh#-iR$_1Y(Qv^)?x-Um1O9o)*}z+!yh zXE64D2Ck?=Nt(ufVV7C*X+Ljs5{$ z&wwp`0$i@GK2LxT?FI0`JOEnD5?KDXu=>Z|^afVZrh{8p%)4IsJr?~NM`P^m(7a>4 z-)tTniYQ z9RV*+>=8E%8*{)N;(fsIp(QE$o1H#AJbLjoE%RQ6Cc5#goDm8VcAW1RJ!7Za3Z99* z`t-eN7tpPlWgKDY!sXud6iv8ASqt8a0Ny%Fl1ok)dtqZ2@Gjo^1 z@h*kqZys9w$4udP7G)56@1#R(B87bzuM4=eN}P?6qMh`D@YkoF>aPhZefK&35t1g~ z|>;?w&AqT!RK9_dp0`uYebE-F`dNuwf5sGG@bcDpxTz4D< zDXy_P?I7Rsgy)C8U4PO!Wc_OOr9@axgueJI&Lq`C&|O#?u9okGtJbK1L4XBoptA*l z8mvh!V+@A(y>pObRYlOHiS~51VqGil>47^L!tE3yZ!yI3b4Ef?sW&hBPqMKgn$uw@ za}rcZDk2~$0Q0X(cx_&jegI?wcqM1e#$=c|1|UU8ftN^D)kc2;OpieaQ?DTTY2ygw z`N=bDmXh}Vc7w2FE?C|M&J>s(&2xcZ!iE2R$Z`-aoYg02^qoyR(5dQ{!LkuHq~9C` zk_W5ekiZ4Ql=$tTsT9!Dm|q;WxW8^slx@=!Ww-4KKGs!^6$wpe@sgL6(GnMdl@M=CE z?4rT3iCxhDJS9diHQ0sz2b%Bb7`329(6zPI)9u}{2bg@7{mjm~`TSxiylg6GX zozQV_Q*=%=U^6uLQd3NdH4ZhLf1iqP7}U0e;%h9)F-$io3sj!#M&RP2KE_Z^J26+5 z#SK#ybrvEte{a!!SbgJKgca8hq;EYw;E2{ONr1|R`$z;SIL;fxQ8N?6Zl2n5gWi#& z>igh&xA$J8Z_wTnN}T7RSYp5+2gXNo!QmEhY?Qz3LWamN8YVg_W;Xm~X8VXSHeMWi z^!vh))`k+T2}C*wSr?)W~=bPg#iI}Nv|B6V} z{=;;#)7mqzVk(4<<{mi_Or!nWI4WIf(Tn7r^hN&X;UX}|b$y8%LH@QqMkjTH&(pj2 z5V$qjg|NX0u0CGV@Hec-am9`u<62&hFvo)8b zC{wu{S*N)4a^lAPft>5j4s7e_5%@b?@}e#qR|Ci#{GIt0co}pWAd}v(r`ZcAFm0QJ z*EqV@>=wou@YZF%d(9V%BGRZp&MvHR8bAnYZh(tM&nV>U$8IXcs zq5C9Vq()Vp&7cArv z66o6)L(P`!--hi9b%UX$JoaJPY@T->Kpk%|&zgsTdVJLWxjfCRd&*ho(LCpa=6>7A z6X%rHDo+~e##ii=R6OtY)Bi>mf(NoXoaJVBHlw@zr2W`>oXfT(d4BE&kIfD5oRp}{ z_$3i5#2H-s85W_I#w1q643+qTIQHN_H0f~p)1dbENdhUwzsi}E;_^@D3sGh1xyq{a zv!6G68~oxp13#4Xv!A_1?3YmAXgox}*IVT)L-zsOqPZ5%LV{_Y642G|pTGsADIEdf z*FTkxZX6{r7tz2p(M3PhQlX4jE5m$XsmOnS;P33S-ghY*F7gJCK}SG7F%PpoAF~_X z#}=n2PfUKUARlg%I9}t6|1?FD@#FEtGKtMwodaE~G3Yy*iAJ*nDnAoM0OtgD38N*!d7xw zNG@Xpx9TF7m=sHNMT(+*3^qLvg4TJ&-2TBe*HUInqIVt8#QX@QW}gk*-nquZ7VLK4 ztHKIu-$PYv-_T26OU;8)sVGO~>*r_A`M`7)jN5~lnuo0;@8a+UZ0}yCnFDtsQ}k?` zeuqQ!YHp8Q4-1xYLAZRE)YYnx!7&Zy;?{RYupk0%nrC1VapQS!Z;O663kUw);kI2H zIya*yo){930L1o%*yI|9%UTAmf?@t(xX=7;R4aH{Dhw@YAC0HB?{~c^(0lqY|K6$d z@9&0;oUgYa{O?NJZwPGo(D4k-G+G@6H$_3EA`EdB&dFfbM!c?Cd;${$vBT!9=7r2ec$#Jcg_IYV1S69ukLXpSbNxLA1(r}I~ID6~h>@_?$ zXZq$8#x{+MqFH&c(`B}t-36y|K)Nr8rls$tBAca>sdkN@+DWNdWeUPTC}x9$2O%nX3EPEvKH1^H{*;MZ#0`- z8KVjdMYFoV8apXr3MN=#DVtDMiAz`qbI=tno8`e+78^?Q};TWU@>C-NT}r@0jbpC=X1o4!rgLB5Aru%;5x zvaA*8E;-9=w7Bycq3~P?_=1`n;^=Fux-e;doZpL43fc!$dIalcUg19>BZeG3JlKEy zD2T)BJVvnYgtwm_K*rxFdaA&h`_QpbmD`ggy;^s@%-I-hLoIv1!`w42&|e$F!@qrn zCh6o6kb3KzWp~JCqcEAJyx)~TmlQ*h$t-It_#i^t%NZeSFlAQo#b|H5Hy({2j>dZ< zT2_Kzg7?2n6o(A{u2)>qgt8>KqxD!L9MaVx%~ZnY31+UT9Y%X}Qv@lpcD9v)9O(!| z<6n>A4tx8SOC|j^YM(O!xDH#WbQ>N;eJpjuRt8C{#8_ZOuJ9_gp1v6`o5^9dl%Yn= zE~GIqMpulk8O4)~8spflP)$R6trzEhP81O(M5!(Lm7gJW6*v%<#_au|p@1I~=VnA{ zI;{thls0==qOVZ%30{D;Yv5)wMaN}T5U+u=UUFG;8F#dYdabR+<2b}*NWmnC@+eSZ zW)k5zB1mlAfj zz&;-(R|g57Y&h=C7CJq8`(tPfcFQTc-f^bNIp@>Qa3v<6y?XY|{(r7!)o0no^YQ=B z-n*{1abt<1{aa6gLwmNRjCj+m>gEd2f7^pAS@xoXjyv zFphI9YQB8)-~ZVA?}OhTKRp}$?PPMZc>TAq_kK?Lr-S+4xW~pj+EDzpQG9)S45JK> zsiJy3fRw&j3n$u;%Rq$(t#wPcPYz3B#Yj2qQYGhq)1It9DsE1#3UD{3J_DN3chm*r zbVkPH3Yn`UU%DAa&U(P-o@Q7%fozV2J31DOJK!B14aLL3P%m-%7^?%Ka71X&DPtuH z$}O3Tnk#prCRKYe&w~`$EXG_F@7i|28l&ODrB4ly*iOn&o3~hVOW%;viEercFspmnG#i&d&^@ZN?lCB1~x;?B8@^j%;fQN~YWo zhr5yEjCQqMw0V5+rd|4378BJ+5~^@>2!bRDo_2d_B|%Wt#UH8Mhx*Xy6og|2%O`wW z0ZZu#n37QYei{a)Z3Sn_H=4ufrzGquk5BQA3!J2vr)ioh{LNmNZklpWi*JnQJhe9a zm2C>Wp2$76l)fxr&%H+7S>DEI6R&3zujdaEuV-UUx2i|Or?;)!L^ArUA{n`d(<`6q zPoGcfv(;g)-z_o52LM)s z3o=jT#Az0m&D!b78CZS2&|CfP2Tg^?ADgZ6WgxMTaLi zwVY^;j1G6-I~|g|$1R%y_|>p?YBQG_|KrRtA3l-1;Rw@f%)MH32>Ww|1(T#ju4asj z{egbJmTn4X$nKzbZ~%GH9k=-LWZIjON3Vxp54x|1U+-HSAs~99S=s$Tu8&Hcr#|QA zERg>^5NZ-moOsImJS9me$tc87m3jJ<1lbL%XSGy0nwYUJT=-*$V^{CXI`A6IR9=Ji z!#uc@p}h+Bl**h1#e}4zV0^iYa>AJ<#uO8gI$;zgAqu8Jl;l(zhM|*FoBh#kx`tWY z6Y4meN6GQ&iMX=nCO(iEfPIg0nn7ND&Ly$p+#obc#_=M=IhrhzF<+bPIw*s!f%O3} zEzum51d*`}C20~QCp6-*`j@0Z*W~5VSX^;HaZdZ-{uw1xwz>L`7@>s3u}PN^3TwWs z?z@9?#%>cb`9GOh@xrfgmPMiD#fls$?|BwoNqyQQZrTQ7Q66?7nD)@SoUss$sT~J7 zj~*h-H)t@`vpgY?5G2QQwl=Qz_}GZ1EMD>_JUV}U#_6ECI!TcXJLTsXqUEve!shYp zp6J^(9@)I%H6B@X`qg`g;G5;Z-EfJGIIFpZWQI}3!%+$IWj1wck(^Lz`~oBaCn~WC z7J!jBIHe1*^y}e_z?A?cOXo(;EykS&@1yx*Zd?adL9UU7iwO@pAmUcU)>@ZGxw_{Y zO9ostBdRa^a4|CK9gC|HnV^5+j3APsScqd2{zmK{KGl*#P+Ez3@cu0N7lu$egMqyE zh|T}4^r$uofOZoxWf5|dN+)a`vZ?J3m7p#psJ^iuB}NvjC&ydzKbla(ZnCzgEfrZN zJclFFk?|b%dWVCajQV3o;b@UN^)51}O38*NYw9CZOqvRsDu2#Y_~c2M3h&D^rt;;j zlCtc|cu!B*DoROArZ*^083PI@Da~L!{6xNLK(0x4$##FlxkcLNav!heCZj9ebs(dLb}{GO?txE>jD9P1^!bGrJif*eG? zbwwd&jUbKwE>HKXJ>!rR=CrY!Z6Qc*Oa=?d&`G%ra3cfPGL{9Xk}}QZ@eaKvw;rjm zI9a>hYdq@lA6*_@qkEl2l>Hdvh_NY?IGEb*>Rry<7CErD0N^K9|E;L*GGma%LFh}c zH4MtQDg>)!r8&`(2`M#!_pi*p6%#$~2ZCt;pDIYIQH6EdxRYVkCL0K=Vjt77$=bx z-425dZ3l5q44Y4T;ElkxHoZ`DD?9Q-6u--H{ymQ8GF-2Hwz)IvSxQ2v>8v0=)E{ou z@?<2zfmVi9-9q!hvNEA++UQws@LmR-Z1SyJjHOdUbgkTzNq)&(11j+%eQ>F2m1i zuq*DxDl|UTKFrAI;c~QqoqzY4czZUUWy*TW!K}=-jvTvu6Ng6kqR~>T#%kshZZC#M zVP~h_lnbhT|}!Ky>7ex2E~PM6c<%Bxh;a}qX5<|WI?-R(j{4lv%dH-B8w!Z za=oh>87SC#cl#Gu6ai%Rhcqo93;N^aM~3`)G{-+hNl31b&>(kgtY? zL?$9K)GWw^LI~0W@QJY;S|m;FRe+l>VjA)EMxz@J6r|{28AYg-;t4c1g5El2(mGL- znuLFxzIuD|=G719$FGk}b3)z(kG@T_XtbbMd6bZZlDou~r#JvJr+gehAl8skF9AvA zDabG%UWgmPkU|-r3%5I|Zp(L3Bptdk%qQaA+t;=d%m{^WWH{2E(I5%glHyvdCpnBUo5mh` z!<`%^#H;Ty;!g9bX6CfWcZkY*$erUBJn%?7WT8rn8>FBVV@qmS4*l3imS`Tp|j zyjy1Ib4o%KBsA)({IugBL1~6r4+SU6HP;#6lW81`ZAF_wwoL`3$a|tRhL$-ZV@4*l zgXTC$jD(81E#cbyRSK~szv7YJs49TyHBq(^Y`8c9vswp2lMpmr1X++!3~{1lP!J0> zffC!46(~hH`bwt9T?!#zn;lLEg#{x|Bxy~DG}BIq^MS_}k=6=&TPsP*aY7>CylaFvoffe6>Y<75u8XwghQ0@bhno;6)5)tBoqINWQ6E{sg#EH{X-@{6jww$b)imDxIY-^42L40;c%c68oK!s zbcW@IF%&O;i=)ju3B12<`|SwuNDB z{Fsb!=NGf2Guw(>+LNcLsBHfT9S**W3_N{?fqW3?Je}mCn@-;$SNF@vv>~74Fj~xy z&>Nn*T6`#G2$W19SbwPyEJ5?1I1Tp<`zZ!M;)9EJkTpXW#@s0i7t{ zibo$osw;>z(KDnnTp$N72GVLWuHf_!L4=QMHh;j-J%@(QRT#s+js~8qMp(osvCQ%Y1e%a@_?sj#M zNIJtedZ9w(lipYJoc6xjRUsBk&5(#!ytn`CIo$AJgCb1Fp$@E}BT&NJNULK4N$t&W zU?xu1oxwk97Q^%#8C`I)26_1y2Kwf&gEi!#SmMs5fZKKHILnSRik2eKI2RC10OR0d z0`4ZL)R>!L)a8i;x?`MC@ggvx;EW15Eup{*1JWhFkq9jVse6fUjEfjddvV-!lu0pQ)< zS|3F+lHp)rwvUt0Y@&3bqF%4JWeEf|H|b>|Ig=Y+)S*)4n{Fnlf}&` zqFk;6HP5f!r(9buO}jJG`Gz0A>{M0c_z`*mF;bm$EB*3iy-C(p5F9SJ;-s*%d-J^0weIm8NaGN+R<@zIGGq{^bL2g zb*?b5J3*1?Fl=OE`{n*dM#_K&;#-VT=>XAcXkJ;&G-IM%c9{yH8ALA`7A`?iWTiW}pglWtW@| z))rmvjM!zDJ3E6L1E+4fGdx$>!r~ik3gBANeS%}gGyC=H!*k?#&KCKDN5(A%S8|A} zh)Gy3A2QcAtKb9*)Uu)DimxRk%MeN|Ei7u-Y*B#ZlN?Ee>IFz^JP2S|-OGSUl6~WT zH#E6~7;9<8ozUcF#Ml}IiP0SpWqU_B8G6b?QtC;+&>06&qX3lpZd5@l%a-^bgjm|M zQa^9ah^0h82Jn7C5{lnbG$U~c_A^~>Or^q!2UM8haeJIGna@e0<1(u7V75s^Xv8&4 zNP;p#DBrEn)kPG7W(`12qY&*427{g}qF%c`)!YNQJpd{))gn30zstxXJp!c6*FTWt zEg@9cdm10!<&3i!>f#d*h#ZgE-agF;h}o33gvSDY%VVmENIZt(Fd{GwkRAhZGYQT4 zo|4dU*xU%`iV2CW6EL0?bhHq_&};)1 zS8W@kV0?KUWU#V<@N!~b(#{?0`*~tBHzopqYwc`KT28r`JcH>rv)>5$d3J=RJMu}LAI^Z5D0xI$?YEC)PSGU9nGYLz&VcD8r_nNR zL_~#sFq|@@8$|-Rb(fsmU8}i>n@e%Ai5(-k$CqNyHzszE5$t_x0haql0EtMSMydE^ z7$fu_<@-jxylezH;XK=SUXKg&%>NxN#+M9EVaSn+;qLv&<;vbq+$!OsIAa(EG zhQR37lI?Y;6~utO$YVR?d^vTO4MEH!utNE2f%xJjiB4xqt?}FFhx=xj)XAW z8XPsmPR@P)Sav2-#v&kPgpx6dkI?yxQ}y05*D#uaut%pnG`u|@Wu_)2m9K)VKMuw- zTmr^p6H~(ZA(xvQx*r5jH#G9$o+J@%xX~i3Xz6AP;ZB#6Id&X@&~}QmdBiPHcTpH+ zh-Mh?Ad;vkQqq)2#M@TEogCYC8x=bvBgr6rlWDqK645mkMUPO2t%dG29vQx5*(jon zV10Xb3{>)bmXXDDcJ?MW&+zLR2GRZS~WI=+{l<_!r^bFN$D>#5?aF<_Kb+cv@tPu2o_mmXTuk7J@G{-p& z=BcTyQ7%O|nUHLZd4n?OxF{Yah8rxhgg?aJf{Vycj4|O{O2PS34V4;$#4WH|KSCqb`3$o4gW}z_(nOcDz7zmq`B$R)QW75fNGC@aO3JI+>)FI zajfmaRNgAFjkF`K1ZQR-FxEra@D?N2!%T;&B&;2|0CL4U7=7nL9Wp^Ai>6Vc^R02I z58T>#78)LAKNnlYan?JZNl@G)L^y+A2RGoZiTEGqG3${PihVx^bY_R5Gb2&nKJ+7xVHxwktN6*lK2IIkD*taec+iR4nf0r$!6v@Er;qct6#E@pbMW z=VGYI*JS!FoU%fvkn4k21-@qE`nT7@imhzM}O zlgl~0BJXK1raTD;9~3j2p(o@R$@;Jx)t;5P4bJ#X2|EkZYQuv`kMTolZiUk~C5peW zrpd_iV5LE{dM=N-zha!e%t)HzkS9mX?d*t!tca2e2ze!Trs%X`cGA5|oKl^>LUvrc zJR0m{as!*LcUeFOx;?rnQaXbQ+I-C-fg~0s@B|i7j;3%-W=6IPh$pKOU;qGII!-+_ z5FVll;T||W>r^e)Ff#+n@%JUe6boSKM3{V?+M{cRL zOjE!vK;n$H=jW_!Z|KSKrwetbr!s4(cQQ=4Ql|Lm(H%{O*_fORU>(aQMDFlrGhu?4 zGqGDc6}#+E#wj(#Y9b~CuN395BX=43;OFi*5LQJANxHCGZSat_mfbRe?=uva9FE+h z+{2kque~15&E&0OJt#JKT^U?YCj6WO0s~w^0Sv=A&9IqKBPBT%SBS1Aw!0!-Cyg@Y z_@D+`1f>dcaJFVcQ=nG$*-vHr{v@b!;9eVPWu54WYfUYCGC_f0N1+a^wqz0AWfrW`j?oD zFLRZX>I!Ex8iVFQCZ>eu)O#LYW{xwilq+zAQI2xTI&Ug#6=kzlNKPC*u~5q{(lMDc z2FlXU242M7?!Ac;Cx@8KVaO(5r0ZdWXQaD_1nWGo22XGiY6Y?gt{qWRsyNPqd?x*! z;UtmG^NKnDE_}^31S9ln>A^~s3=qbV=lro+zBr!hSm-rig}d0Wk1h&*EB1I z-ijazDmND6IKeU|G_9(~+y^&D+cJUR@lR*y)nCtcL?V8!L45JUi7qh?;%H2E#C;8o zFb@j|(P2p3P2(WJs;$utKpb;J7@ZhLB>{6wbm$b~fz>&BdK1(iPYe9ZlpjlCE@?~ z|NOr>0LbC4ud#x*v6JyE0*y&;IOwr|3}an|f@NOYMZvHaq-l>`#bf)+$o!(jfEk3< zDVhgR{=p$Z_aj!(RRk7hHf|H)5-$KZ6BkeL$&*lmbXl8-rWr$uf^cO-u5b@+r$G(| zh+J2JXd3gIy;eruS>qBK4_dV=_e+8q=?brcWXurbx=|4>jorsh+i1{PDxCvka<8Kt zbGc%<-SGg@x9=TP*gY5FL2&QRhY&;olcQ%t+SL!9bmy+rvW%ZyqHJigS4m@zFcuU3w|jlzl20 zF-8_K&c7v@poVY3!xo4Mkdh#cKn2Sv4C-`{_~68RP~-xfg8R}I0A63=Y{Y9%G41T4 zP8{X%|8+peGX~MZ$B9%gak2A)Iag3h1lJ(*iCg*YJiAS+x4qeijNuxtpTWR8R?(j~zww5o@uianYkkvmnVQGTK_fNsNOR zB3gcDx4|3kuC_|NVG|I$tO9;Y%)g8n#>h(u~hxg=F&VPp13%9OzECoW{BENpdIVKPPo zH3Y=g^!QV1>gH5Rca=m?I8+%|EFl*30gqHl_k9`F7r;U~^fX|jP2?RLTqcbW$PTEK z+YreT4lM3WW*WlCzWZQVu4fw3`Rg->pAzt#!8E4X2VJKwm9Pc;Q_*1t;Kea8Rm z2O0+!mRx+N2&bqNY zQWE;&c~tpp{m*vqtDVQ{Z6Ume6|1t&gG+`BvG`>p(~h54wr$kb#wVW^lJwj)k9Q5~>5DHIwXNFD18Xu$IQ;G-oSMZLU_a$hjP8q46_fHnFi&F@E_f?A5T!=Iz9z z0ZuFRP0h5xPG3!J=b(m(PF1erFO`9otHr}T-=!3dgn~7*SR-?p@KZLov&wI&tiNXT zR_=KX9DgGjlLKA;y(F7>~&!p>ActQ}(usrR}DdD~k^IThN>dz7PSK3{-c8Y0P9=Q9eDP|E5+!NL!wmwyr4M&`E` zNsxG`@2#>8WJKII{*3Z`f%E>s;r{clg}wopA7mvdB7)K-XeQ zCD>&i_5~wPSx>};9}^?sa6o6dd36;%5SGSWL=wy|%VM&00xc7=kxIfuh*h-tRh?#{ z-MV0q&TmqTwnI#DHizSUMy}P6KvRZb0Pw`&j$#Mybb*X8r8w&{B)ZTtTPlp9% zVP6cPsu8i$wQ#Z9aPWMvr>2BBkr@2$ib|d)2iFj&S@2Fpj8hR$ge2TzUdK>;#0@&2 zMQ{=VlzVJFF(UBf2NGfiS${lW8rQnImX}DLw8#lId%@QL1hYo;w`48-gWnimTR$eW}dMP|vcZlT*F2V-kda z3F06bt0m&TK7$MOTX3kenBJF|$6|>lmVjLt@d2_x1y#fBiZhu+V;E2f_9KM^gJ=(7>?uU@bVHFE8X?DBXBK8Td8S(HF=YbN$v7HI9ty zDfg%1WJkN^*`Q5s4e3}L@kGM2lckC9+ES+5D7g+43wDVuk!^01 zxDITK7JwREIQiaX@ez9W_B9XvBHa8TFP>t=lmJc&Q*Ga11;yBZQbkguUFo40DvGB# zkbhPBk6+EyNIJYgl+%cy`T6TJnJbhxWwfqUD9b2ixYJE-yX<722s-V#*NsVrNxoEM zm^e3Wh;&~hm2!1%U#KEtqlQ@jn0^UL2={=0yfmVi8T~c-* zEZt=_SS8un;l1=+<4zbI!Ju_P%-Lbf!hY4=leqvaJQbC%%JeAmJbLoPqN=oxrAZx4 zysE!;g=A^@v9ty;BQe&=?d>6}ADv=8d!7l$?vEC)K&pw?WQur_WKr*3jzzX)v@I_7 zJ8Zgs%h~maOsA4BD$lNLXMT3nVRpi1)_eOJDb(OjFwQ!ubBE&u1 zXxmV1z_x}oz#`{&a4F)!YCwwR6Pb+!ENxv~z@R5swPG&y^JtD$rhQwoGYh!!cn~E# zJm7?6%F_rWje7^PC%Zi6$|9!*A3@Xw4rWF$Eg{j7&MIR9#$b>T1+#r5B5zwFHTC|) zC-XUz(IIo`^e@uMv5A_6pcng+{&DbZpy_b85bKTTrA1A zSe9W;a1W|;;yyE^z+%qbg&e*1Y z6Qs(V+k}_*MBW}zm#>Z>n@S%w#8l4xU_jYn2e2C@8nV`SlQWPAXZlM5kOJ3?1`#y13rT-eb@;qmML=?K!f@sV=5fpzE(@A+HHW*R?80b4mCupTe*v$|Pb?A0BS2v^fv{NA z>aQ&bme!wenp`eMcuZrCz0L4wJBlO2R$|Ej*HFDOw)Ne_wO;2zvz|ECO-`8hv8vn0 zsrDW#PJC)qh)s?Baj8*ROlsthM~%v3QKQm0)G~&fw)vFGMKQEIzFDUA*rvrRB`V4T zh(i^1HtA}&%gapSV9KvO0GQj4K!x8e3>>w5BTkQMWN!=>9mGR|+=vXhT%ByU=s!Wc zoq>vy^?UL9gdGdOv-iA=CDmu%&}J-k$Cuf(M#VectYV)U4c^iIG`qAe&X32m3(}}+ ze|8@-2lwvn>lN5FAvf0(L{0vLi-FQ0%3ygL2Y3%1D>Dl#r;qT0MZfZ~zkp3F)BgOb zSsEUGH3&nMk{YUsJt3UXB}gGtNlEx5*I9B5Uvn8WPe@ZalgW3G$G-B!jazV@UEP!;r`*cruiTPN@VzCMP7DV->auCi|`h4n;MR*(oFGP&ReVQ292<@SITGWvm+R8Tv)OLSfF2(8unUxx_oX(gjrN z*y$R&sKWqvj##NNj;?Ul9k?k(Bml7iSmFIaUMRK5;*N+=u1H5$L+v?lj=mmzJ!B)& zzZ&+%OE)ltI0Sznp@x7!2M-mNQk5(5M?s zKst8=;aSrRWT>1|7vm~C@6(uvcCiwxalxp{+{(xa8#60jv=#Fcb#gMH9T96wh4#>G ziJRzGDa6P9iW~e~5?Y-ksWtlnK=IX}WQ=-I3PUnId5ojBJh z9U;4>jBBLnKP2svT#z$%(hbfdX>Phl$0T8gQ|@G0qu4PIdtrdp{+49;HU)MZ>(4U$Jx_+$+t!<`zp_!!8TU zu`iF2X$dHEA_>NjENHr*=uDV*o?&Vzg*jwVqXgdHjg(fcfr?C6rljl3OChz+oTa{e zRHnef=FwC_Dw&H4jAdvR2!YbrFf6A8BXB7(CPZTtS!x z`3#3#W(4aPjWZmw#T>->&KBbEgd|S(EHXS@K=`4AC<{aFBSUdk`cos3<6is#aqxo9 zcsfsh&P)h7dK{$L@bQn|ez*URn^F3BeEE9t?e{N_FUddt>m>gEd2f7^pAS?Gwth&0 zahzjO^W~fW{>R>bAN>CK>DlOSCzG4S>%WD)_jA%e9nAN}J@C617O_eSlV&7bfc;vG zg9KDW!k!LH0lAgA;4*U*@JX0QQ<(`2E89FPnug5(kx;Q<ZVfonGdw@J1F=VGTjW{CA~c9*W_=jFRw3cy_%B(k*#?76z1u5Z27|%Vg9G?~Fc`T14-WUA z{%Lsd?BMBecrbW+@TbA>>EUqyPiU~NPF6h&b^!couyR{&=RT9?!w2-Z_hJ@gv21vHnLoa4{d8iQZd}^>L4vvj!;&hYwX*c2II?~ zD$H|*A|ClK%uzQ+9TEBb8^G|b2!k$g>}+lQJ<226R9NgZF9HVSG+Su-`HWoam`&(^ z8M+12;={?MWmr9MRX?NMlAQKF1ANMu=U|XSDGJ|r}XXK-a^4>qVn?c^f$e6KxvG7yr#Gb*Rg{&Laf+hwz$<7bo33{juOZQSCz+j$AAz`Tp;`#|4OrHezurQ zqW7rN@Ayjs?s1N-iq2qwQ_fpYJ%9iEUu}6P;S9&(A01fd@DcpYF2y*Z-GgV(o;~|| z|M_t1UCs?|KOi+A;~2pe1H7*{Njh>H?KOVbN=eh>FeY3 zSDhVnd%FeMX6u1WvtSY=K}Y@7n=IlObYcMJtG?)@oL38s`1Uv?lucojg!nz`p@I64 zFGdjX&ipz=A5osh5rudXot_4xjwz$`p86^rFpj=~_QX8NuNfrm(Cs;uBs}@Mhm5(C zTS<(RMq@jS#uRnnRPM-M?Brtu801v9x3hyf9HEYCqa)7B4#!vgLY)^u8jPYiq7js4 z-^^D#4r*JE5JYwXgEMyV#94052o`Tas`!eI!=BJ!YH7RW)Q9ZS|DIJi4(DZc z_N))A%?`Vfb-aYar4ttRFO+Y?8)4Vkpev=7OdRVAUGC>RJZo9^8@01nUTT$rk<#Br z15zk#B3S;hb^t2j?iIhT(&1CEY;4e3tDuRH|3hsjOX2O=UsmZ{jA@gu8O7B5S&)S| z!Qo#^z8Cwq-u?I+8F2#WVL<$CU+(8!E`r{&4qUhjXlJIbG@oB#Z>$+Pr3_zo`K}os zo?v3B)wW(tVoV#MAzQcO_w=M-?lhn?jnQLPn#Z6KQYv=TOFIhdi0rY|FADX(i&S}Y z^H`8%LV2WnpAd?+@$ZEVSAi?)NT>Y1eBHkq25~wIhMgVJkyZfZ*s;Nd=v?zFoV_6P zG{`XO{P~+2QnzLZmfxj%k=&6tgA^=Zk%-Kn}N(kAC2I=sm#Xq+mp`W58B-c4Q zTci*uj^8`vTj}7*eZIp`Z6VP9gTy6{lEr(TwPtsV6&c}}T#MI+4FH5~mP&Ip)+rOD z?HMRqNRCa2{K{ahBlrroA%@{W&X?FW2z2&Gq20}byI zcDCPt412=^w)Yy*9tOu_-9z#tJ~%}xT3X0=wKV54<)CBu<{qV=c9R~B8Egc+7_O^!XFCs zrYRop^tMhWk_>PlLeHG@sSzRAsqHol7S9ZMfh~6om*(l*j5(4JWE?%M-qj|Yp#H~t zX~3(!gmJAk7}XFGNlFNO4@r`v6AHGg<}Z1Z!st}i`BqW%`k^z3)K~_ z*4x$yY6)NW-|~(FjkntfriU06WM1f7F`h2@ki^*_L&w|<<2lJG4B}C(7N^C+i_`~OZLp8!|{+&Y7^9EfQ{scvh|%5t!1_?7w^w?zRgY)My@c3CG^a+ooqUKW_+k3Ju>#b34FEv&--vT zBN@esI>Xo66FxeES9a8Da$75OlJ`<@bOx&=GK9k>4RR~W3NHosuwK(%L!Asy zbZ+oDD-rOvxrJv+~dO786{mp2v*TtX24}0LY(0%P1#Vv3ynR%W)@9}v zVaC{&XR>pylnMIZ5 ztE_PL6`AawpC?|+q%;LsYU~6N2SgM{EyCn>r#!R);c}5sWMP>vVj3C7shYHc)N)Mb zqln#EfMd&>N&^l1Rm(uk6vV-IOS0pde3g~*jHx4@<9%sj9rjMR7zk@CW4MTH^v;H< zD^0xWhO1Dna@guhuXFhLdgI!88c-~Qr|TrB?y74#Hbt1mC0C7*uu2oQXPsiD4i6|H zhoED1dfD3E5fq{!ACNW+(KD9!x+Y7;-M2iO4xhCMmzx1Eh`XQz#(02YQd(}@?bIVdWdN;lT6hEz_uj074wdB_Laa#$obr=n$Zct{2& zV@-!hk1!h`Fu~kG+pvk&JkWgglcwzBa$@o8ROSXna4gSHfkKgeN)bL*zWb@8FtjSy;DP0%AD)>i{}q zHf${ct0Wr|tE4^k{-vWj*MRuz^}@Kp5*up972*W!EWr63BAMtcVA4LT)v@6>2bZvU zcFi^mOkq{Fx{9Bz1-8g>2CKSknKhq&dyjC}DCNY%iOL?>;3}LYavF;&a_P*Mjm_~j4A+y?4$Ld+!UY+Rxvkd2{8NSd`{2|#}G~^ z4QM2`2Q*0N3}@o%D~weaIYy={=jkGjd3cn(C!|R@f@(5;1|k#mLBzLEg>TV3NOQD( z`88XhH!DR zjk&O)L-rX8rTD^x!IXhP9}$8yW+uo3%=qihC4^H_D2RxUOv zwmwgnR*8f015%2@A*;{3$7Tc#H3r_;8?5U}_}sj$o@&7r)wBUZgK4Ys`7Ge=@uaw5 zTKL$&q;EJ9HtU2;r;BcHk6Kn{$lu=n-}x-qJAB%cPr2H+$(3 zAQcyGXbNwk8Oh~M(Z*Dy(GUo4TYBOuP=g)7wfVx+3DXK)gct!NL)!&;v_E9lB8zt# z7Z!fnC*jUc$xM|t!bs5N*Lh!n$or**RU@p5$Tgxfqnq`Lmz>+%zF7QuA7-P$*0FE5 z7ME5^lSL!FltA33K7_gGY}u698XRM7ERRpGJ=@x}3pO=$@~IfSs@EZ%@o7n89@SmW zW@E_fe9)qPDdVaW5H;Rsyn-O_I@5M@;x6kb#|s5Z)^8Lgm@Kib@a zcB*FkJ4c=!aIJb!(V*_pjua3YWV8Yk^kJO?pvJ{Jj2-;%>s(2H)TC?imjt(Zp-WGp$^m` zPAHWEzb(cZ(wZ={_~tDq^KfT1{(A|0oNO)XG__N>EODw@P}ehdGS4!{k^fkd>`71_ z(zYCKDGA>QS+U^s^o#c6XF{tRJ0tpGEl@KLk8$fAdJETmhylJhFPHR=j<0hXj`T%jniq7Ti z4!3J1Zrhc)+R#5uB!9W}Lo|w%!dXR`^2@CybXb1Md6LQMDd!m`mzeWw98G7C{CA*+ ziM1hT6K8XCM(%QnR4kIZWpWn_ABLCi!*a>)50$7!TuEpspGTgN$1YC=79A4p!MmatXL-5?;RKfqTjf}7y z{L-Y0gZvIMPo5+cXID}w=>60l+al-{ab+9joM)J~5H;Sn$I=11DvFDImM-GhJD~2z zC6M)iUl!vD$;S8Tl7wPsIWJov=mF<7 z&~8b~xui?${w;CE4ti`bmwW4N^9*c-?UsAB4cZo#5Bq6u{|SM0xw@yF`Db1Jzv022 zlmBmT@7cj%WB+*{Po@2*PU_kNQJOR%e$1D_55fjr2IVpI5^;waCu1nL zE9HpzElQ<+mlDMkuVz%*bO={6>?{LR!K$s()K$Q6L>DUbw=IKM!7x^edsSww zOMJ5nZdYufHCv^ET0>oQd&}b}G6)~h=;;AY#%>J7&y;kp;MxiPT4LxXKlg+j0AUN_cw1CTKy~8Z&&$mP{S= zrkeEV8^qH$E~BVb>mqNoC~^Z`3kii;kVYpW&;^6IT6_!*RunpO!f~{0q8QD?ZpvhE z8c1AyAvxHfa>htFjrlnvYrEVrv!n2=WOSs*TCLLA18&E!Nw^c1r%6v^BbIU{`{q=r zkO3~sIdP{Rouk&=AGPNCYHRTS9htHFwp=1m4ab@)+j4&qq~wWP(U ziclA^pL~s^v4HT{WDdES4a|PE@{v)eBfISsvcqz^Who51P+T{WqQ`at%d<(Jo26vP zsVpn-t)7;YvW8ke;+S(am8A>Fd<0c->zahebBY$gaCveO3PQeg98!CYILQ zkKNl^w?Lg%zUf93&-p1T__AmhJ%#~VBs7ZU+JbYki|ng7g8S#HcYMX&nS*7qMJo&# z<3=LQu`E4S1`BH5zL9YQZh5x!q&T3K2lu1yzreG{D-38{+dFdB3ndERk8w$N8e#91d{Hgu zp1@NnlYc!yBoS2 zLgfrY$%PNKx%8pRiy!h_{_dtbO=J%hTd3>qX_C2%md5#+){vKd^fPE2G>r?Jv*;4B z{=Mp_QjIX?BP%=Yqa;2sX#v}gjJomqY!AAdkFV9N%Ki`rl2^v>VG~F2OhKB33*v@j1xz487ozcS+zNXT58Qo(GvH3F<8sWPL{a0_ zis29Mz(T17{m;{GMq=FF0H}`sKN}pl_CLdeXB+x|A5T;Iziqfl+p(U~Jk_O4IHFiK z3dS-sq3=_gr`)DpM5L0$plzr_=Vh9saBgS&;0aR{$zDD%bWH$0XZC!}|X8N?LEXt{Xf6@gdRVLSQI zd<|OB|0Qhy|5Q)^_YU`5`aj$oZ0P^JJbxJaZ&%QQCA->P(EU3!N_NKaxTS|#MfE$h zP)@VxDvI$Ghev1<#5p?`9*GpVW0GMJ$pF!t6z4rNUESX-pdBgXHhOzItua};!Uu|5)v|C8AgjdN~Y*IO^>rV$xs$R5-K_i z5_I+oV3oJENTp{^EG|xF(Km~nLUBHxpgL9ZGC99N2zloqb3g0ql)>q#XixJfIYK(n z(>!>8gq|KA?jLR;G|4%$ai48wI0%1C;+waG&~Kv{^UOMOh9Ba2-u7vJ{x9YF4NuMa zzdtPKe}+$=?QPEg`*=Q>IdCf&fE~1*kfy$0iyXgtALTSkL6lxO17)3cd z4dV!h=z4||*q=OhSws@{b%x_PijtfLaSRBu5(-!5@c zuXoAw3*#+Lg=Bnewqrsr7b)so4LhjAJ6_*_c}dtY@!r{ie9a}chOiei%=xO52@H3R zXtuyRF6Yz8gWeplHMog+?3_AmK70ABKO^L_bA-|icg5`P!jDBd&4LgY3wL=Cz|Ils z?04+XQpDEfhhFChU1FS8`flZ3Z~PknEl;I;^1NS%FMZii(A0>nQaNT22gQuTVtr<3 zf--sXlMxdF_u?z@#n<5|RpLe;ys?1P;eb2KVKr`sPaW*4mD8XJgN3MTt+t^ituzj; zAlEFgK4|s03ZE^P^yNNn_&;vnzV`SJhkKs*4~HB7k9&DOfjX~J&Q#93!0P+b!Mw&NR_102T_@wZKe?;=H|Bn6G#-<=9eg}{_BRtlr!1F}M%ObJfryE;W? zXu^}z+Uc`vwj2qWoa*G%No96&+I%ETAS=yFOoH$)K^!DwJrMT0`q3!DSOJMMxEZ_! znNSfO^Q+^4J&wYx*3GNBC20l?pj`X1VxN+daAmSF#tEa&3%OQ0_H>_T!DJGRxdgj@ zkiFFkcNp+MFcZWt74Owd#NMEXKadct-BmdGM7*f)l`QG8<`ZrLwIEW%&YsH|nl-J< zs`fHM$(Y1P=={a0F@_mt%LZrNYdqRA?Al!mmv4r>y+M_Qi#TgeRXa0EQ(tfb-t(^q zUk?i1vXE&+d3AJUwm4eQeYd6}G;a7z*YvWno#mRXEbE3M=VAVjt>hlg%ld`mfl{Om zbvwre(s>a_IH8J4IN$jdoR98Ul#~*4%0u?n27WDRnuG31s6888VWug#FO{b3u_%L+ zqUTd{&0o&b`u-Pi!-zK5z{hp3MWl;mb%F`HPv}& zce=83>@x_-+Tb{p3t$~rGaszdd&zRCu-%s0CBm*qsB%~7RQB%9c&s)37X{Zx|A&LY zfh+&pKiJ6s?&nzt{rAbBoI^k_uiltsnB=`NnfKkJK~(Rz4#UyXfK|7kcjATmGvjV)Z(~ z9&O<(4`pL6pR*#$%DqF0eRSC0w)%eyADo0K;d#R#utcRWqDBnEps}6&J*DYhW2v%H z2N}^0YuEusjHynv40Z z%F81$!a;^JU~#^UbZ8Ifbz`(jz%7p-jdRo50OC#v$ptcLSSwJP{yZ(gkjHN6Fp85! z^RvZd61_*AehGOih7Yr1UiasiX3;qB$QVvpG=7VLv`5L*h=km5ObQUpPSn~V)B-a` zXGeQH6sFL_3JXJUsmcho2%H<|2%3)md2tc&99tI`zetf22qdAzk<^=9M4_VGX&j8f z=G0n?H4V2%+9Z@qqs>V9GKiavFTZ87;=(Xqwca+Fe`$E})Vh;)e90?$z#$PuO&@Wo z6K_|9vuSl|R3AVpy!>h~{S(#{S9PpZpHQuEEr+=puHx(|KfqdH>4&x&hVC)c=9p`R zeA9+j>yAFTYR|(~*n8oAl*FOwDG9qYptB`AS3H2q54hslsn5JtJ#W;gZzQTt#8%xl z$!a5#-Ku}3&>PS0m*CT|d73}1<$vps|GfX~sT=?0V7RxD|J}>84*6e2Kq+#74JN#oSK0+%pAzgg={K?i(P^onodcrO zw5|jfWn?Eir2Vm$6Y^SJ#J~HySg^7ayVkU|b1K@_b!Iz-(86g6h_q1Mv9WV;Em+mL z3Io(G5Z}6t#H@hMISnYDEaDtf2alA5-6AgCdO}p%s7q$G0hMo>9h0dNZoZQL!|&Oo z;VkcuNrt;&k}u^2QfL2jP>BDrKRnpPf4!H-!~e0lewpM2)=$1ycErwiz5H!;Zho)M z9SqAb$yZm|YCs76Mn+QBE8Z@z!?oBWsLG=XD5a}$YX3kIyk#8|9&2o8UX&QnqMQZg zp*x#SamRq!L$1pJxpdp6*^WTltt<=sZHr4wyCbYabc>&tU*}y|12EunS_Mt~Bg;CL zr2nm~fNJRf!9gMZ-@)PGCjQgCJj>I6+eTo8W8nU6gEVP($dRsS{0WS9G)+jxBYB!; zAbBN4cQXa7YJ{}TAOrRoSVl!96YaAnl2o+JrpOT}uNpc^pTqU%pDe4t~BZ#CFR2_6^qd_~=tsv4t&5362$JENwP{ zPj6SY88-GSpTd5n*>b4SftF>lZ2+yX9yYqo;?t1-2Vxwg5$HJDIs?@5|NE~0*VDn_ zM*edzkB9$fbA6mfXE4*BBYPjD5nrZv!r%*oj0er!yCYz+_xAigJSM+At6tTkBE#-* zl569NoTXMd%#BRrTS2JiH7h|i;y7DD@CG56QJUtK&7gkMjH9f?!OK^ca1Sj3sdc(8 z%jjlIS{-+J&-~>&a5q@YJ7F>_Cw56zvoXxVGI7RP+iXhxe6S3~83(;_MwLdCVfBNn z6R>es4fk|$y%eJaQrh6N%P9Rj&Z5apBT&U^9Hdu9Q#ppVq^6;aplck&S_D(n8C14h zqCn7kfltkz-TBjo{~wctW+aYTabXmw&eEzYKn?%DH`qUP`TxV=M*e>71>vkv|Cw1|37c!?%HKV8 zP{Q4%gSO)+r>NJWPfc-eG|Vu*J688BW&cJr_D6bFJpUV+gVmk?&xVEbfB1B`IsfnD zS@Ze-ux-OW2@jy}L-)x9ENM%&!~wnP@x0pM{6!zl4Xp6Kh;HjX?UcxLsu@&U5MODK zQH@Bo`C-GlwZ@1>jJ7Jm;k8bbR^Q4^VMTszJ?DS599zk7M^ z=KTMRZ3D}9?{FH1*@D%1)s>{9SN%hLE+X)MvI?Y6eK zO9EZHW$wm)qBZu-?I&s{cm+F(RN`NxNHtdavg7Hwicbqv{x5EyAukq3GiulZm#Tpx@lZWuMrvFPQfa>V~ z;h`)4+ut8-@_*mUvlRXR47@z<;tX(fMDYGoD~Vdu>b8`+r5eV9WGi4iY=7b8YH7)D zwN|gO@fUAnudmiZOXK|RLp|7*rT=XdKy~*2gWB;viiO2i_}T@U=&6zS)qm#)L8V}r1Cu~ zlDyq-s!6Jz27%1{o75l3Yz1|xubE|R?VG{bs7OC2 z6{&AdSI~9Uf!ip%Hc!RV=Kc>Hz!D0;TK(UC;r>4yZ0!H<@%U<~2(;U)*94Z7Z|S|Im&IqR z4qTf4w@?7q(Eo#H`%fMEe>i-)vH!T2XEpl&xoZIB2L6Mq0L@`2=m5PF{*aY`UWC`7 z1@r-4!fDKlUkg28b^Uu+1nNWZp=tta`Jtwh)L0kTsP;W71N|e=R2x`XyHXuk3xx-` zYOVuZIjohfGit8U@nXM>N~a;9jZWut(&>07bp@qPExfg9b=vU%Y=4G}xM|uS_0-9K zo<1$ae|R?B`2XC?Q^NnB$*K0T_U6|Itp`vmc1?`33C|S<=x+~1C*YfHQXs!@?0VA!Z;F)zS#uQsAV@Jii{%Fy_gxYyxWIcy^G|NW~*mZSgebO1HvQE7{Wd3jLJu&sotm-LY*sFle{DK{Bue2l~U|JqogbKo%beNsaS6JlVqiWM746> zYX!$v{Vl5qENyZbWnc?n8-?KKq!28f*cFt5jbOG_47R%e(STwZM6-Vt#8DW~214MP z``_Sjzi|KCAM9`LfA{hD)-Ic79*Ebvr-`Bop=g_3$4+w8i4s@_opJ#7S}n`(TTw+{ zdVajZSr&y@!k7e6%umO5)vtpbWq3}maM(kqz4Ux`+m2l~CDTuaMU)C?w>aHr)T23le>`YM})p!&r z4aT!KVrBBW_Pe8Yqv8l;JRE@s3&U8{4C|iD)4JrE-VAtSSxtkw^~SIsAdST;4eBn1 z0AEVy(HxTn)lv`RWfN4qP#k>jLU2+kl~|jcY(^52$vbELU>L<^-{BP-o}32rv^f-A z1+|u0cwkmYE4rmO&_;0dwtu>jR6Kl1g=bDzkWth?Ync_o18sa9UPZaogS0sY(tys6 zP+y8K#sLSkIU*k95%G`+?dN%1wcAgnX`I^ShH2Hp`lvyZ~pv`%xMuWkdow7dwR;NI4=^Pn(S)H|!64tzAhzwKZq^r=k6C zLPFe?KUky8S;aOHXIDy{4WQCP5hU8 zdBpioGTVhl%zVLxM!pYujH#DuIlOW?1X*%IvN3*%)0o`o((=H5Jc(kQqoZ$7?_2g) z@4J|cP)Dz_?yxsJ=zrDwEhkCGxbujLWkKY)GWbo9zK(Lb6;V8g3yzGE`8W;eY?lLh zK@y4+3P@$)|M6-RqU@ouh;fk6l1{6ykA^^$c1P>kurwW~*Al3n)v1?524| zD26)24(EHf@jRt%tMBZ%@k%`L58C-Dv2(R68gCI1=dr$hKjlrEUODb!xV?qdD&nt* zt%s>tyB`rH|Cy7d(G~tv6R^#*($ntz&%10@ammlR@_!Bo2YW96zyIvnCjZ;LJWHPc znkwq$P!JM$SpZeO{_tLlE381mShxkBRB^I3>ST_COb8y~Tebw~cxRS+*;b#$8n$&9H?P4@VdoIN?-LV7KrS?mA`rME^sy_D(ra~C zHjN>UF~!a*XSE9-N(Juap!5k}l&?b@HuNrf%yls*Qai=?>;@B~I6e95%m(wZuw%;I z%JZ_So&7=HX}KW^h`rL$Nstly_*CH_STG*dHFaMvqR`9g`r2MZtzOGyKuC5 zjVpd|V)F@5GfLB=e!o3cG$yRQQ~r_B0|SuI5XoqHfK?XFPcsbnjDDxVa>>YoVo{@l zqaKfOo*yAI^;DFLxW|nLSk=($NjJgRTRp zlNe=NEHr9sTxCnH*0v>xa&==REASpxRCb;PV|)_6#5s);^CaU>MKL|@U4yQ@Yq4P~ zfcGqjCPt_jJ$>0WD1QeUM(K{b+-Qz79I#y>cgkCw#?d&)onuDJHx>zvV#b;@q0pIt zwo^_;yLO(C8fx0;@aN>TD;jqiFlk$AVNpk^jBbuIgFO)NjqhEO;w+#zJbFHOKJXB$ z$4gSH8sY>>t5PDv>F3rfojgeSk=rQE3z>jz9UG&7=Opd{S7n{ z!eYvoo0xKV=Jt!`X$-L@U<)nRk`1HvPSc^6*npgw2VLdG$zN}68S}oZr;+^6)yJ5# z+$C3R)8OlJabUIl@9^2dz_tH*x_`LQ|KH2QmVW=Mt?zKcxgj*k$Q(WK&hQid5uAFD z`u!}p?oA^)TZ|St&c>okcmCL@`E_D9b6za1f#iiDPmdB~uhf$>q_O>?-f3+qW^9&lgEVZ~8MF&%5JU zkkPzPGmQK5AWHh;h4)D6&w}yg)>nN>~$MfFv-t+Erf9a-LXQyA@GurK#@7y;MB}`<@y!KM+6;!BeQT%qio3~T10k*`; z#e7T^8c|!YY)AdGg~`jQgg;}2u*3CzQqi{m+~%5KA?5y{IKa-uMU(&5-!-%pi+!1$Zu3h|aKem9GLl ziQ?E^cHSc2WkDF>gl&YXVVGu7cuH277+BML6$N5E#YtG$rG;`q98Hts{M^4A{Z+1} z4NHm%Xjpn4yl+}EN*b2TqUnsDMPbvvp<*bw(%AW#lyOiO?{X&r9`00f2>Luf422muIFL_X|6SSGpnx*<1UXkbYFE*EcCRRr zvWRvdacUpDH79+w0~|i^MyBv(i6=p1#R1R|+7U40)GsV5)2B0=So6TK9UYc*!qUjK z;8g<94*K#rssG@R8@ZzYj3Q#ltEvVB53vb{8-u|pU+NG2 zheX+_ziJ^H+6F4~&IleBmsLZ~v(m^eR29>nulT@q^{vDdhNDfcE75mj8YFU!oz)1Kcb)#GZJjF4tdU;O;WG9VIomPilo>u zFf_U4kJE@~c~-|GIT{O5kZ`q>tR{Gq&?*^Al}BJMt5#kygS9qYA(5DkHQ z28__yPK8VoCir0%U~EVYrmt*_RBIzY=O$2AMG0OY1HUY~3*+h>N86J*WL`%AFqSgF z^HbvdE6oYOspuQyK-k79QpmKFjugtq0i2@&dWLUPM9m4L_X> z{fas98koizX1I9S(lI8&@ZoI;dg2me+vJcFsHIEMogz15UpV84d8^`YmEQ>-lVh87 z-LipB+#=hMIYozyZhtG=t(j=i%Ga2-i7I2iRO}B^?OO8mF*>>rtxKK( z#-O@YMFe@`+oFJJL5b7K<_koG=h{=J{gCbq4O5ECws5SPS=MX9T9scr!O4u1@W6(S-&Sh8jd zo*KFEJPoei3=ofc+rHDgqF-kpfql&io-1p2z4)OVGcP0fQUz~7qoPtbEcV5pIIL7(gq?YQXmh$6P1!aFgh)%mfo?uH4B6|MC@2?ou zoW8>bSk(EOlEI0|{Zk1KFulhm_1tdrA8p_Hx#uhco4O7Gy6O4;Tl6Z|OK`%Ib zyc1$dCEYHK-k4nAnovSE#!VvS^ZV{*FBPdstmFvmmmu1roQ>BLY2GOeb3gl=ItoPo zU9Sk+kD_$HyPNTF=_V2?ohYw(z0m!2O810<)6m}6)Xzm)du0%i_+YKP4{IgTk@0I* zIcApcMogdCJGhlB!Wb#p#y+sWQ&s>BDR6*0JawL}YWwzH@tx)O(|QxA&|rr?Ra#`| z?eMk6fOE;eBMup?$9C-d(LF2IdPl{_1@##0>|IRLR7aj;H#+~l_^OSw&CbO4$iW|m zj?>rLgFYgeeB+jHEoWGsu}O~8Zu9bf>$}huTB_WO7JA(YpFi&dKL|4c4Jxs7HPMHw zNhgelNchj6Kku{f>Za`p=%)1H$qqTZrbcfBBfy{I*hP+E(CBWiT+XX(3+18%&PPgs zSfrNuh)6h}t0L-Na!axBnEHzywE zMao$+qM0|k}&I$X&l%th8D_|c@83(-3i57auEh6T1OrK@c0)DGD4Q* zhY?HVg(cD1L<9h2Cgc1-ZFjKD>`EE!^z|DVvJ?#|#$BoFSZl;=u^FUi)!8_sdINW7 z?A)m8A^Ak2@5L%z#QbTMfuzoLa1?FcR0R%Ss=@*9c`fBItCRh>%e?aI@Y1vBksG}; zrSCJuGnG27*+Y#FrZ=Z~^0*rD+vp;jS0j@FV*!G5AwH=buOMLkg=kT@`A!<2Y%|#A zlHI7T@hVDxMJ+CYgR8eL`p`Mq>m)tUOrzl#f@M=#S0I`_57sBD0(PemC6i@Ur7jb@g1ly+x5@aCinlRJo>9g6Bxh4~IDeWybeTa~xZ5DIlD?f#XDKj# z7E1j)xZp4iSZ@v+40hlk+zVjJWJD;BWX~%$3=qgn45vhxVLBau!o|6mOEevi>M3c5 z2yu!@MJqK=)=$uiP27tIJz9cq`nO5#Cv6+>+qpGzI1jf`i%Uj6nAm3!3etu*#$-pd zsYaxzWo2ZqNSMuhz7p*^o2uGYAWeu5VsvXiy-IZ(%gVDIm#zazjM1I7&k_f))F{fj zMYgG~0;BvyRwvy;=v%H_S7QYhg^2XK?(fVgpOq7iq*0fGyzH))Ble?+0d&OL=R|;w z>xc>I-?*>OF2t3wo$ek8N*~!*xSVSix*qsYD#yWKRDIV#RD$g5xc!hcyUsr$jNbjR zT)UzvP~F$>KiC-G6$)@Vk0T)=eLg`T`MCb1@n94G^ege4BPWPF9VQvtg(NQq`7MDu z7^m(8L#d><2b;Wt-crFUk4$0%&^=%N%|sh31pKM+Jkt zM-Kf?pY4&PL%sJ-Z?6jVltO(IOz)X4&jtQ7J$Qn;sE!Z?|0vc*^(U0^K_UWm+f#`7j`X_+aS8!R2}I9rp&c%&QPvVlCf;Abq40 z@d(6YDCJ4uT&t9B-6vb=v#_Yjl=DXzK3E`BF=zhLF)qXXm@Tl-0x;xT2+iGqA9GVc z+|vvqif_JN2OMi^iu8Ib>x=kt;J%jLT_dF(5#Zb;{GM<+##A|Fwb(PF z^?k?<0RknPRjvh!;B=AZ4lqV$4lw3Y=Oqs{nXHOv8%*nV#JqiBElQ#c_3;}&Vzg>* z_3_ge5kyVC{xl%073}+ZdA)mESxMXR>;3Wev2oTGzeSiNFm(Pokh+mi)J53%CXQ+P zBydNAU_HwY zi0h9jB<0&6xV*dMQ<^*S!pF+Ixa5=ntz{g0@!J&fCPN9GP_O^cYBZFj$e0 zUZ67M{*z}AE__tb?9}*6`0VY$AfPS+b~q$%nGP7S&_p0(5uk}eSZNT+h_ElAYLbFyV1DgjZvoL`cE9?Yd_pi+` zo!C7j3^@ilyuakn66DJZdATQ+E<< zobbVW?RS6{v+v1aDGFuJK|4*l9_oo_vlDdj2e`-tVze44qRr0a4=RXM0pmv?By+%GAmLz0&_s>(_H?6$5|D3_z&!5Rtqp*Zf0P&j# z>IO%jQ_+ILguSM>aubM0dlp6&tX#TD}myf ztda$we6IIts)o3$4y|xmLONdG%+o4RMYIW^8PqHWMcc*9jejoyd}mo=FN_l2|MQD`El44 zuxg8n8z*a=zb5PJDKvZ!ojKEa!c0f*P}*t_0pVq*{^yqaX{3(+4Q;z6(t>!y53 zE6L*rRs|p|unQ{PoSKE^F4Zi(4Xj_4)u9?bjahIg$#T+P_EoPF2IXYUl50?XDJ~XM zlg*u6EKs4g=)1F!g|`o6%GAL=BiZIJ`+X$W2T$AfeldFT*h1@Mw z&7aC5GH8!A_8cs{p7R!+08haHM}a8RU)rGgj5d>|+n1!5 zzIm6{cUMo&=S>V>1p!VU(Y>gVEqef=x)ArG_5!)tZ9Y}iIC zRxabOlJ`OcG13k1%`cBbZ6g^Qn;v<(LY>W(h2F7LYTvNR%2TVlv9GF1>(4&Dhx7gW z^PkJSXF1l|3j7y6id{Xw{l37J2~B`N@{F#2cHVYH61I_QjCbeMA@mY++%SG)u9evF29E%2oUZL zU@yN``YjVC=xZzu_QA>F=eVi$>`<(QE|k&k8GRYV3IkD-$FTF{5Ufr zd|NSZfzc=DK}TCp-E}1!VbyX^{g}REL_mxWtI~zKL*?TY#T&q*QoV!cvm{)jMaGSx z&?bgir$~3M_=eKuDJC~bb+b_*78S9+^D9{(AsJRd+Q*Q}4-=o%7Weqq4Mwp|(D`?@ zXKYy#?Ox{Whwd!_^xWCvkAovOjFT^qUj)sNHG%1!%J+S8PRGFRR-2a~Th@@AiyREU zN=)KT_VL{j=>{FkGYs&E^w&b)?zv!UeWZ9x5O+}7kLdN{)@`Gc8Q+&NdxH>6M)+Bn z>8Cxt@zN+Jr$j>3$;8G7;O6b{j^U%SQ%ye0)TQ z?~(LP2my1=VtL3d@W!l*QMr?9&R;=ShRhb-)i;vcbU#E^Yn5;XT%$cX9xu;0_e~|| z(Oj`=uSJbJs;%ydNbj*fkh!9#yK-&U-t$)7{%(u&8m!5FFZBBE$$j5*1kjWU;+)oN z`JtfHPCJzLy%Cf8+ly~ySK$v4w`>KV0|Azux%VkgmFtf4Cl*XKjot1Jsq!h_W>XEz z_(bemesRv17pPak4U0KTA_SDFDN(>I;b%c=I_cb*h9EAPEG{*hEM7dtC8$SIqAI@g zG`W95_HaauNz=!Sqnrf95&yF6#c$=DZM+yRVsCK(~4?x^x3Yn7?P)MhJD_1QT1EHyKL<5(I zd6i)W?bD_tx`SADcM`1NdInmN#Tjch&RldyWK$Xv3o9Z_NM!`-SjZY#&>9m?7@>X9 z8l>IEO?u3j;^~4LaUxTugsS@Bz{Ygce~K?7hE*d3SF2U0Obxdcyei5)euW>6SI~%3 zRf$sCgvzRqSuDaO$kOGUSW1o8hSyE)Xpzn~7UEThXC*pLHvZH=mY|NxLyORlS|=(Q3sDyO+px41W30Vau zrwTd^wXF&|jut7Ivy#>;HCX&CRuA+V$Ojs8#4iSYbPR69u;htP$?`P2+lL}T@x6AM z^#+p326a~xoFyu=Kc}9fI1=_|pjhI4Mb!+%#hP#zDDYW)iAp2^hcd4*_xXdF4Q>y{ zr`972YLwb)5VPzG$?I3Un#oE5F))Js)EqP)4a8h16+A$(KzP7h($qkB1^l2A(?hKe zN)X3Tzm&LnI%7LRfQ3K;lJ(~>kb#6I^{YV1wwOVy$l{eZ$?P(V4|Jzg=#QYevmNDr zXLj?!_&ry-dQ0-gqd1hJI%K-;&4ltetrerD;hqkwCCTgBm7_Xjx$a$r^4z#3$@}ya z$ou>*M%}S3N8RB#8E!kf6svSwJ{`&a0UqOwO&bT56pNolJ^%{(0IPJCQo|<$83g5e zsr-E)ellvv7F1tiRl*--4w-bxu+xha5ARFjSXgt8J{t?2nJt9!M0Ck8G@OK)CKPCtgi2K{l`OckH^7A z2A>LPE2JE4rODXyl8!m5sUTv% zO+L`c3;q+-07%ZcU}e5GLB8kLR8LP4At-x%Q0azu0ahZYftp?63Tg{~PbQ~MvO>h3 zv4joaRRy6(&vQ!lJ%0GN4qVf|=s%zc$qD=fkka(+Z)1Vtz~QxgV{S%-7)ksDxYBmd zZ)2e#!cf^%q~3h;$vn=FVW|G)f0@@6{Hkgf`%t>DzeAeX0)#oP3NfrcJjbK7{f$! zRFj|b?mXxTcYiarPrvs#YG749!m0Fnnzo0q(d1CZy~aYHt^6_w_ulmMG%-PL>oTh3 zWh+wtdvj=kwj*yCk&4X7UZ8Ap{#%}kZob4^)q34WyC92Bi8Dxmsf4{u)tX^5(f4~l z?pn4qyyHfTY|OX_u!@e0bqSz`UZ<9+ZRSwjnnGJ2Njxj5qW3{!&mKF#~M8TDV3C`lqBoH_2wck<^(6|H$mM&DQfjBZ` z2dVkdTt0DzO+gNEqQ>Rdvpaa*5&VGuYJH5d;6O$ zcPS8WiZR6g<0lc`s9WARF|lVQD7Pt?4`&h*-=h;=7Qb*2Kv;erX4O$xeyhQ*Fl})# zOodI6pH`tpg>6zg!G47pM<@AvSjLj?0~^Qbm-x>?I16)AJHqPjKQ0A0ZpoSrvMk2=H1*wk>6Z_ zhyjNuvTZu#;1NkbYELEjc4bTZEfA&Xq9JvYckC~HyNj6mpG2x70BWTz)7vEU!NHa! zX?P14r(JfaHhQ{l1S4=PIp{WWs&}is)+*YX$$s0^OX|T^Geb^0@P>^Iye_?g03Ec> zo~47^1wYYq_sjK_)eE!OsKEFOmc|m1R&tKO9Pxix9^5di`zYHI;Jh$r+zmk(r z8}-)@Z0!7p`;&lW;2MW7M-K5bSPu`{nHJr4LH){mc?feh=spZUj));dy0Nu!3u3x=|0d`m0rSH$m^t4Xv9{IMz-dy~6M2S*O}kMJG%asHL#HKL~hnDGy4#ADEXgmFCp#JHX;+_;{|>va6v zW43>eACS<~X3NztJ*0ZzV?9ZzV?9^kA9X|!_xi#gi$l=I_`lvj=8N*5%J`@dfAQvf zU~947Gn{$Ez*7$zILO*+h<14uiKTozp|Ohjy86Ble2Ae)KE(0h>io&zlZ;_r={t6< zuSE?u?yBKc4-4@eeJEFWpB;Cl$iZrEA*o(w!QXN4nt1cc<7wR)Y@KHTSt;f*FaG&} zyV1cK!Ay+Nkg=1HN8OCK3`7bALMjQprj9M#4nOP;$I=XahiyJ&vxPrO`v{uxEaELu z;Rfm!{~YlJqj^^*-il)V2Di~=MkDnKmzoH#BkjBq757bD+ZwD$@>bzqOb37RrOP>o z7;+%%#o2L0a2)lQszEXr^^CopvE}vGQ(&em^+z{K-cy;QNhu6OCze34!lWx<;LglY zJ&iltPU0c@K0iS2&$w-w*;1oUmJWpLDC_>qQv0tQ>j45Cv18#Y1-*nweMJ>pNqx6{ zZeuWR$Lb3FE1gG_Wxh=5!^4J53Go7y!4n$_ww}H(pF=(P!Sn1|a;dFD0SCp|ICPW| z2V|Q{($$n1F%Y(t+#>Zk;SBXT0T;!IK4qaT6&I$vm-b5tUT0phZJ!IokU{fMPiKx< za5|ROk2$-bFve6H-}3nIU?0+4+$aTZb;0wwz$*%GL%eo3%dPdueiD%x-4>8 z+$h<_)I}zIczSG{l7a9w)z+drrjLOPwJsr1fQ@~~2{o#?6t=~Um1*3j=@f3%8n{$5 zaKduLWRG4e!{(0}zQ^XZ#*4XTi4-y`NcRIB?ZjeEa-)!R=B$Ba*R;Gr-x0fOqBdWc z_086L=ay+ddj>|vRU1djz0R`?w4dT9RQPv(eAj7*hv7mD0FjoHc$5GQ-DX&Fj{FJl z=Kna2_|&T3AY>fVTkVNDg!-M&BTTwX>tRVbhyjq>s(o+7mK3vWjb^x>a9f1^9Dx-B z0R+SECUAtXsQXpI7euBlcj2=kRrb6NRA=hDQY%W@ePUs#-hT=aWg%?7;isM=jU!nlBtXJ1 za&IQsKt+seh}jTe50C9>4g5G6AIRNp>u`;|_k+fb0WJaE}b;W zt*G@9+H|>5xA~Q#4D=?c{CPITaU&B8i#f)zASt?}!5`dYY!IH91kC|nBaH4xeII<76mOuhO|v#j0mI|)T)dv1 zmLs@>u zwhJmNf2^O~d{tIve$}}C_bQfsv!PyYY}kUDu~k{Wo5aLg zg&~6b3163c+#PG7Y&922?BZK9%XtU)wICg14G&9CdLJ)clrXVIVJpp@id!CT_Yn?TPOo@+3V)p^?sTAczJ@V#Xzuj;)Gk-rm7 zE=~c8+JrydcW0VC6 zkRF1!#>y)^=PkY>MUUO1Dr-yMTib3R{3Ve1+WWH$%zUI_o8fk?UP}HNU2cbzRiJ;b zioo^`+uJvqblvzpQ`dA1ZPW#@dwyo#TU+g_+lfyi7FYRYL`8-WanM*g6Bql-;$Gqp z6U)_)Ld9!;zYBDKygB&X_BOq<@6)1-ZSN*iU|M3i# zmep&9>@!u8Qmn+~9$rM2jEX0V5%>v`@l%jIxVAzKnh>J}b&qC&E2uP;kqtWRT_926 zf6QNXY1r}76}kr?1{Bi_kP5TU!eRLYWqXFJlJwh##k5-OC**&<--H$!l--mPT^UV7 zW9|cr8O4O7ZRnTyf@%Zfo9DdqmLK1HV+c`pqYm-UCwbfVeSA9y;rbMmQHxoOCx)Zy zm)!PgDp%EX$1F@}njXw2dfjbP#fl%67>nuol#e>HD+f$oRC|Xmf9z?aqS_gzaz8uY zM?77G$=3utt&ZXf_<;H(7ylTS>c(Z2{!eRy<0l%b2-(X!{ zd=OkDs5vL}gc~R%+@+;kMWsUeX!cL+T(Iec?xw0QTzNL4&+KpQ-yG5U!LLsj; zm#k`sdwYfu75sS4hoc%T_An^wcnIVPgrE~0!Eqe1YrXm?%NnjN9!*}67(&Th0!3s3*skM zfO-U*e;9uR^T)XzN7&o3qj!J=g;@rxrwCvN=8s?(hKT6T1X>333uGszfUpS)arLi; z{*M=F^DGfy2at0vZ-optA!A}49t|Z@CJq&t$KSXPLxx)WaW}-wLfC^OxJu+c1aZS} zKD;~?2})cqBIymO9)5q4RWEW*Vz|AR4uPdmFB|}07>#0!L|%N>Ae?U!}JSN>w@u6f{- z6$|}60DWEKSng>mtlYD@mTr+@XaAK@J%EG6kj5WgU9LbCc^1bE{|P?5Ii#gr2-tW zgrPeUUqBYVDV34N3_|=Pn3aN|Yd~|^`@?xP!cI4Mk?2}eUf29CE(ix?J$Jy2(TX9< zDqjDg;=6i1_qaiJD_E8&^+eG+9)!20OMoIIlp9bLXfZc>Ed&7ae`&foOol6}3z&IS z1CInz27HniH;_EXx?l*s^AQXAeBUidc`{vkVx>BG(ugKFdPuTgx=5w2TtLPk!_kZZ z>CDVRvd|~nLb8{g@{XmK5EW%VippX+sOF*TQ7@R!yM4Y`Vcr!cWBC^t zAhTQ~pps({AuXxyi&|N%-^W3#X2b`L4O5UoN=iPy(sb{zH}A(fE&7iV?u2gB{o8Vf zW-voZ;@pKH*3lNIfcI2!K_wmJ-UQ?+cIKi-Q(<>niiZqEfru3ah#@Ik$Y`Ivsneb# z%@8>$qAjToFot+eW@D<~P`^@aGfy(rYMF+PgQsEQ&zYo4dW4)d<2A@}B_cqI-jR{6 z1Y)O?;^fOCeoiWPJS+0C5EuMXoFqtKvN#OJfCB%k&GNR&<{x2ErF*;YSv)Vs!M;w$ zN?7@9vUsR>_dh6^m|~@2g(H#uvaP1ylnns&e}Ks56ImvbH|x~_H!hb^20D+_-8SfR z((O%qN>xo1SNBZHF$ka@2aQurCTLF`)0kac-6yBye7bW@K6V{p)B5IA#8+Un) zXIblVRu??iEQNN`t2|#lhV(4t8$Q$5gjb*+4);-)#JN`Z4bjK`gdF>w2EiFVP_lq| zf)-f{~dE=7A^V1n@?nW8ULYURjy@_~X5=vhwh&>Or3U;vWwV_UpK! ztOwJ&&D(w5KX6gb-^UYNU}ziG?Z5+F&|ad}k=(tP=Q(tBcqkf+YIEvW+Af?w_6NV& z^aCSL4z$^C!^LI%s5$>A9%MpUD&BVI_zR|Dg$91&@OAo?VV)P**I7WU1rj1Nu*UNH#VTu+91_ZFH?7v5;mzwH^D(*WOD8H>wRL+_EhWJEy zFiPDidy#3BYGhe2FHU6}{9LZpAX|)??^(#Y4jHOerGpvt3pCRUAIaTs!ZVYV_S!@v zuu)t33aEWaA~184x!zF{B>r#Mdy?~KVeg~i@5z2-d#Q_H+PTE>FUn>58wUKT5dz`K z9fO~+O*O5m;@9F)_g}o0i0i@2Lb7@tE!}cTuD5z&#^jH7
@)aAZxS@?!&ql}$G zZR-Q@EiqA!Z`i*lc+UwmLw1q0bn%G%N_n0U0vctYJ|bJl8P# zkpI>oj%pqYv~sQw&?g`!H+2U0MF9S8l>CkQN!cO*_Luo1BsmqO2X3Qvs9pSbAD5@6 z(DVzsjyXQ7_BG=|$_+~0(Y|S*o`LbAJMY6VoU>vRpb0$3sNLb8>@8jBOM*)oijA(NmI;sP6_ z;<^{As@)S%gznxSCW}|%Un(2Hm734(OA~9%3!T9xiz-9rN9-F9-{>yKEl%lKy@j8P zm*z=w3h!M;{7UV1JbU5J(^Py(at=SlJ-qQhcz6Z~))eD2dAhMwo%@B}nsQjJxWT_p zK13MD|HhYW3#au*SP$-Ajp}+5>6rZtOar=1%&TC$wM3tyBR!JVs#^trM=YI0BTzfs zfp+#=1b*TMr07gK?=S0TH`J933Ua+Im0Z_fQD~3 z3?BzdvT?)rVx{Gpck|&WW$Cg?J+~P%kI?9-^CPy~P2ib3P=zO$D!avZUqQ`DOy@mF z$fow)Bl;}3>%{0t`UeQ3g^B0>i-cj^tC4R0LBh7V-$+=6@Bc-@myzE{*c@jsCNq)y zTu?wOwqxW+fmFS4_bMW{N5-^u$zu(9VdHGn zKk@`6&ztcVF2G1y@aeSs>y)DQ4W`_`{jbS7@WuXqH%eU@f} z)}fo5`44+$GIf7p#Mp{hn5f(}UuItY%U0Mwfr>k8dbH zAbJz|A)_8a9_P}xPJkvbrspk(-{7G1!(5*hTqYcF=LVMX<6BT_a}4T_qI+BhLpCfnb=BV(9l%Mf|_U6~IQ34sPr zr7fo4=P5eEX9pf`ADJhrTp5-h!L${Q(^>S`$NaktT$Q~L0{?ZIboHB(#45s~8)!ln z(tiy~h>CYJ9)tQj(f_pEN-u||xZ)?FFhF(jRj5}WNb0cezMfBydOD!asDl1eYgn5D zv03K^a5$R=>$eZsx_jJE^eRsM?Q=v#(tlYNwgu}U?k0hw2J&J>N{J}ow0WC~g$q}x zLOZEev-*%bOvilFZY09Pr$jH$B$b>HBvn|9o=)aI*f0nBSgoT+h*Jbr4d?qxiLJ~l zK|+8)S2wqhqrK{ZO+hA{|PNs|(1t7u59 zN)SL%fqBwFX1)do@$z(bg-jSpzEF1>b1=)xm*uz&b;dc&5!z{@mCO(AZHL!XJIwF^ z0rj3f6`*^%lmD$>#@6mBfp_cNvh{bpJZ4>+T@pouR@0JxFWIrZ56Nf*#ReaK2HEyS z>4rsUK{$b}Ji6A(FFMj1Ikwpx3vJ#|V3*D~B4ci5e zOti_Y(D4(gpjXCrF8D4NOzr$7onwUP`{Dq{Rb0OZZWfIE9lxa~%15>%Y)rH&NB;iW z3aAFsA{J67xy4~8@F4SczFcVdkNIxDfaHALOdEF`vW-G2LO?EMN#Z~I8?;{L6Z?35 z=r8lqya&sd(Cd8=YT0Ex5Sb20B6|jrkVU}k;RR>F?6wd*R=@d1;7q?@@~5qan((vb4(iZAxaOi6B7`~?vOjHUH_FEys1L|5qYd&MBKMlo7|!9NF(6IR9;bqIG-=W3;r-P4d9ya;6l%FJPbX1qnZAGl1VAtCaoQ4~u* zyMbbX@mL)YxgZ>;FQZRf2AI872(Hln$YWFY=On5FdAQ?W(j8Q5ft;SZGWTSU_GX$$ zavcK|ssB`ZJ4Xx(0`r>0A64oq9TtRUT5l1yj&#Mi6yf;QbT9hQk4Rn&W|&G2VyI3Q z7mzWn+fdX`C*U^mfVXnfaDq7YzPL}(e?DKo@YbI6ZhF*DDuZ{S`!VC~G}`{ms%MP5 zm`gW%iYmMNR?c|bd6}*;WAyH%;3oE@U8c~;SHu&1aP-@50;XqwOLqf=-p8#-s&}(T zYP8m7!7guarPgM9C3n}Ar*@eWpVv-E8Uj`Rgw`7eEAjSXH@UZ`kvC9OK!VZl z=@HkJ1;HyL@ZjWJVBKVSh3#b6@4{frK-&U$om?eBf>PzNpw$LhR)(8|i5`98X}tMg zDCUQ-ln#bFj5{g_?<;bytXqV=5uZFEhjQ$_?Qp+L_M=?k?U?`O7rMRXVbiiCU&@44 z+0E~TCNNK!>&5O5jz1N?OP|0>z=eC^lX-p$=g=BN?AWT%)Dr?hY{ocgvHYb;)`8al zU|MzhA4*_hKBFG1Va~HcOEyYQ&Zno!iOt{dG^t)__B*HnwUPKf4{KW?VwD^7gZC+Uc{8jyOK}~Y53sppe$GU0IO2-sL8TD_+oAeYf}V5y1I!TP<~#{RDgC^Ww#SzmLG&y9awtZ2Qppc?$fkVACbbh#Utq`gXGkUr)THcmK9b3Ie(k|D}-HAe`zw2^Sg?Z&QP~Jn~2bt+g-t+NA|JXh^ zRJmnv6sekjv0jqJwzU3Xd@*Q$NqU90hw0Kyr&G@-?|}0E^*fr{$#v3|t+J;2hgmKp zUr33zeSK_meegugSXB2W%>wokpkK)5$5hyfe?NSuEum7US^r5}dQoE%&a@9a;89=8 zzXYIi>;;sjE`^v8{Ns#>j)~fRw)lDP?J~DIZInUGfAhY2L4592*7sKQ^ytw)mHf;( z$Q&EPqohe#FVYPUFt}zQ9@K(R6 zHZ@Y^tDLC(vxN(Q(Vu-081XxI7XW-)C@jO7T5RNwO+V&`VSKf`Ln-HnKhs@1T^~jg zJIbpoay{Tpqh^G zSC!5?QU3TYLV!_dm0A2Sg;4?hsi~&%N-|JkS$X{!@!YEZo_{kYrUl8BENQHm^!_@lR zq*Ge_^xGvdZ^}+grm`)No%lp@dF?MP@-r2jOG^^Y$? zqpNXtx6u?F7N$;@&+6YBVAIBcxbd3oNk7#L_HS| zndp@H5n8kTw|3og=$S`t~pR-Vy<-X}+sNxEv%HIEzw)FYU zEfOZs@ckd3XfixPDSv*_B*WXal+Xk*Y8QX)2z!6$FCmOiC-tuEHaFJJfx6i2Jp%1p zf^n^JF{xnIJj1p?qnPkS3Zg(mL_Tu7m${BB{&(q1rXaN4=(c~z;qf;)bb@uaccMOa zDMwr9;o;x)S3VPEBKWN!X+=FB)nLa9=i>ff?65nX;2^%J$r zezVK+i#BW8lVsj@Inn@cRuUXiOcbAEN=bk zTj+iTd-RL&ew~bfwuM)`*S_Opv>3x|6P@vHL?`{^_sXL$J*2noi0))OUz^KF_x7}^ zYLOQ{m>!Np**Dp2H{o*NjN0^AsUT%#O;uW5jQFuVw{22@f}9nnpwR5s$9eqFWs#8_ z<%KpK`0G+f*-26nv?Ai8THVmFCqQUquvX?mt$n~9g|(zmntxo2(rf0}k#DbdU%Ibs zE?norM65+u?LF{E<2xk*vD@T)QT2!FDYR5AOx`+6&O@=2+F)&N)e?S+vSjvPOtA<< zqUv-+R2>RmH48|xgkNfi;Z68lVls_ZJ|*I{E}oRqQ+qLTcOlCuedZxqJVRz*;AMxj6^bH26fmlzVXEMqymZe ze$x!H?ytA1pY{E9AJ&K~4 z;iL#km)+V`9X;edHBx;Wd*P4FAB(xkXYmyr=m@UA2cx>GTmx7V`h+sZ-7GniBN}&b zXi$&xWrugbCWl*TQ2h+?DPY9gRm8PJ1ZC}j*wJCECI&2u(57pK-wKhZ?~o5{IHBdM zk#}tsci3-we2mD_=mM~tzYTmLoeKN5k9BFSMp_LGX@A@Dx^R@qN^}hW(@ozg(S=Ep z;yj-u8IE=(AvqJtiv|@B4N%EV!zXW41jg4{f+$zOPp{R-qU68@CmtgxwXn1fQ*3C*KXDuVLcI*(&|1Bj6tbWYdC}ggRAK=_%?!KTE?z-q+Cp3y zEICKEM!r^-C#5I!jtZy+_I@~@IjB16JTu2xYN(R zSiF(#Jeh4sG?DBs*++nglC~jBV$fmoI_up^&djujp8~+2pwLcyPK{Qlb>1i07N6Eo z!C;`G)p|JbE4{x;uNx-?UOO)o7c&Z(ZX8G4`1E9qJh4n$F!crEt);hCza~u*9O=He z(p<)EH{}PoMkQ>GDyC+IyilzQVaen&1&*l~ezNWdDsmQmG8?o*__7IkVTDGISh%&d zDy8kV?u5rAig`=E-xOBnt{t8Q%CiRT@4MDvmUSRdT@K`mIh&Z{wM8>h#GG2;j)V@4D^7m;aIyI{y)1NkwO3eigZdj*v%VV;!@J?k_?S{@ z?Q~_OpDY9BVp^iPS$tpc`9Aoe3>Qa*93S7Uf zG7!ZS{Wt)H%7|;?ieT>c?}t#3m<{C<%BHsss)+FC zO>8?^cFAX5UHQhH^L*aNlG3qxefX%g>g^n*Gt6@(US=s8>Ydf_69P3nHJ~{22U)@j z{S$jhiG!x!5ZQ?(r-KtaNLl0(e7q@!(zK{E8r&r}jXHFNm50z$`>Jyo!HX0jYM0Xu zLn)Z2S2fByZRXGiU6iT1+?;2$xY=mlZyL%aj3Q`hNc{N7ivm|Y9aI(QlMcdOf9$$b zE5CePE?ZnjNX2!1n|AT%UVF5kihC5lT9!jzd*WPD7QUt?9d%z@HZP+qq~76J_MiWF zY}Xe+U9r&hXCZ}7I-0$mXq+25ZNVvs=;@bF=%6{ll6%R(-bO~hf#)*0h`V+2 zMLSqG+r^|`eO?L|c?ff;v&{A7B=$R~7_rbVgBlfM9-|w*QHa|SB3)bQD;NidlMf%B zoNEhIyllCXLr>rYPxWM3t_`x5rGqlet5#{Ei=HmqTH%V*#P@TOrdn_fWUsBIeHESK zPQxgT&yHHz4Yd4RDq$9V{KQ;0JQ^exH_MT?9B$HEk`o)F(%EY04HeY7q^$F*U3HyU zF`duRy=HzyMU<}egdn+g+%%3O#(^!YjaiUx4D=BViVg(3dX(h7A|0?s+l5uDA;!!v z8tUKpV;-x_BnN$V+EFV-QOxc_BUF=Q?UT@DG$R^$Q zB(z(#c~{%6gJiTYCC4H!TR)(3X`g5; z;l6=U@udE4PH-j!Nr4dmwt|QH*p+-Xr=fg0QG~=s%xO9p=fHU1TxznERb2(6NQ)qg zY87tKqWVZc2+yjmF7!zE5{6mYY}PvwtIIP?=NdBwVo$9Y++K!3CM|)JR}lk)5*{+U zhmwUcP{>(B4z9r?((z`C8*PVxvL6a<=RzZx#y#S`0wD`VzI#jNfqOw!eX)MPBAA+e zEj@N>`&zTz<+jmN4oE>>7*Z%=)s-?_43D=`CNF3oV|_z*k?GQx=d1ueZgc z9CcTX1*b1p+Xf{T=72tviq@Tp+=R#u&gT4`@1Y%jU10E-tQ+T3ayG3)%pN5PC>Au< ztw;GgB)=V>^ak|aD@W9v;4RSUT2x!uvKJTDB^0e|Kbw+C)>Uw=ptFXtvxG3?7~jdG zXjM#VF5zrBl3QOfag-h{C4~{IdIzJoPy|tQii!k?6Olf_3_*Ga)dM36&2&{IeCP}d zdtn19faG{_GJkJI^%>8Os_n@=n?Ob#omK6%eY|dNdf$`M{nD#Uc)M3%1|&8UmmF&< z1PSjbPXQ*qUdOv(?>Iy<*d8rrkBBf#cK$B0r#G3_ny;}Uq08EyqO-K#MV2LEUJXbTD7=BAWmtB5QGHMZ zN0lS}3&``B$z&R2pbM#z#|9c9iPs}Vv`mgq1577Zrc~vedezcDoXGSX8Dy~;C+hN| z&VfVJuh1xN1_#YH=M07cHQL*CEzh&`32{M~VzD`2I~=eGapNH8hGw6TpFP;TZi)nBI8sSla z@;M;7;+?_sTkHjZ@27P@kNtAn7E2UH0;lj1yNY0XOOkg zaZme=1)64g${6y<3p|0|kn39$-xQY-VQOJ916qv->VGf1Xj1+oD_zZR&c7&}^J_Z+ za=eFhSh@SyqD<%Ke<+xhw2nbZQ*l8IcxsIR<^ips1A_77_7E=febl_2r(6?ccJd&@ zLasf`@M0J*+QcXfRqHcz!}w+Mj5&*4eBgg@;(!= z3c<+&AiJ9`<_x%z;rnT(tdTv_UZOhBX2Sdtna>;xAFC_^8|&U<-^iJn?xf86EsU_! ztu)pR(6?!gp!3obwrCQ+W2ZtN)n{wyBZRH69{4IFCWr$it|bq(Gg>4Wk;Ni{wv@|E z#wryGqAiAKXHy}4P?`8+3V34I6qDPXUGKx>bs{%MKfNcBwCqA3aea*x?+rr z+4;B?zaa?^ExI=OlXP}m0ISIK!LwRt;bJ8HQfxYS%sJPFw%%rwW4eeK!gb>(_KcNw z%7G_uMu36%$~Y?!#Vuryxx*nJ$-zqVb>+Xat$i~bpsVE}H?plsm~s1yizi!gT|U4OCn zMv~hb5P?Nr7yPx_7ggQCYkQ`$+eAB3I(bvhcQVMs1_;$!x<$LDYyb7Z+E0I^N{|>G zwYaieWB=oX4Y^WOQ{%gMcn(lgoBC=gt1R10`X5f%Tz7@ne|N$PAD;WVUBlN)g_pyj z!JyoiI0Ez@|7_;NSInGPfEo6$S)TGbqKt=~l-=em7GNhJq8owFJM+xRG8EOmBjkSe z4TiMPJp1{o6(H)Z2T+=rS>0Th)E@lvERQzpLwPWoPtu=h%|_J6V>M2OPhBqmuiXKO z3Nie@p6virFhO#weiqWQKT8zqN8SGI=g(45gnxHeCqpC1s zG0wdUC&NtLEMbPYcoJPBm{&Y%$A9y|y8rdTmdL$F{Ro2`%r>YuqxTI_fG;$u<-^>( zkd?W|R$3=w7wI~#&=t93c#cJP8U619kjUu&6o91vO#nKcOLLu<|!4pW85 z@yd0%+3)aF42Sc0m<|D4s|B;e$ZO1_nl=8RYNXN#y`F^z9QTCZW65Gyikk|snn=V$ zL+o0={mic9bTD{}AWM>A^#>+J#(B%ZnIi<$1gYG6=#Z8~n%l%lAk7j??GvN$BzG>U ze9TpaM8)Hw{ipmLzqL@AQK}r_*+647ZYjlfgT@2_Iy@}@o z!Y85F_^TJPI|Zq3Lg{~}dQ04;+1e$0$%C6l5ULG}g9NE|`X{j(Tz7|ZAU)3J4Z4`h zQau%6L`?9Nl8kf1>Yd8b{f_kgDHD@ADb_A&9krpamDq4!1u(tWhK&J);Le1(`VaPR0Ar?m)hklC#VVU+$f>Wr}t=x3?H)B05;1YZT6HJ#^cKZ zu9m`!tI*XGai*ewFEK>{(mKGggVP$jrxsNII{^tz1+0aBoQtLP66BV?cFfh-4lQna z%ss5=VH;W@*J)&|EN9*NhUu>e(h;i|rh)fIVaJh9)dn;mTyB1ri6d#432tK$_fuM1 zNxl0^iA!Gp;k2l*qfg>M^BqIbqa*Zk&cu)bf1+>_#sg2#D{3L|RunV8Q`%V2D+Tb} znimInT!dHf7bF5}9z!V^2 zg^K=C1K}3(xV$qSwKg12PiC2*P_uUv3bgeI)Y-Ln2alW{xFO9TXb(8PJ>d8ofaC8| z0FK`_=Bg@aAGiV32Kc4g(2twPIWUQ9FQEIQUqKli@(c-rjX^d*TOl253&)z_DSNvt z;Gg7yFM&4zU+E7)Z&$jh!_F3|xCzLLOw9+d4^k4eB4R#)AfHu|9q#XMVcxJF;Kl|w z;WCbGlCc3_vB-!ckXVw?0nV3TCBiO*KKQUt4gJ{_*I%iIq{^`q<@~<|xDM`By_E+l z!G$~gD3P?ubb-|-ynB^yvGYv@$3i4%gA&brAt=3mDmmYH&hpBQQANUKo5Z>~r?`?s8cN22sHefJq50{2QrGDb zt!+>o)?w1wYfY`>-B6(33r2Rda_?o6f&JrnArc|8x=1%G0Y1MpjjuRX9>dqfxMR3s zv_*EO-uXajUNqFjg7lE68@!~wayet z{E6T~J~4w(g?w{*|9xbqMDw_Q>8uC;YS@*LRvY?7op!B*0yRaa?fKo7S2-q#ov2;f zF~E22D#vn@U{?M?AM**8hQn{RxoU@_E;NKeX4w(n#A?kNaoLg1hu^LnZMh>Hw`Tod zX`H+YeP_bihFq%e`+>6br)=s3IxXhTMDPT-A24ib@_PntdP_%(VS|-?T$^p-ii0L| z$^MM#m~WSl%;C*bI;Rqdd%E+jMT2hlYS+On8qXwRgd4cJEY*wp%7y#yl`Y4eH91YE z4)x1Dd)GQunK7301*?EXFKhYR42g@ecCLzh4Xf-6^WBd5VvD0`id6`?5~hQlplC`p+IP!0namT&blt&;+=naD@@Y?5ukEtM~qlw zueazFt1iz3jm1)Qq-J1&!8oqW;i*qFz*v@vZmKShd&`4j1MgP~*HmW1opPn1UaHib(l5AVk{&&K zD$OWNqX9rQ3rkbIM4iwA+Ku?a_DOY*SJTH~ido`ztXY2*`*SS3_H;P|KUMCV9v*IX z`-&@1GlBt=WcM8gfb&Ta3?@=AB@qKb{NW}zf1o4w2f$cY65XKj2hUpg0%_y0jP;6y z8?2SC2%`1l*pUb2EwE__aaE}buo%ICAzmH3;f1#*?x`>x$|dSjXuINSn4n%ces}=D zkjcW6UJ2q>(15uR;_V-WaVjj8#dpM(-IT{aZw42!*rAI<-B7`Q>l}2dm;(3*44oUg zm4DM?QtTEH{*+yULkmyrYG`%N7gq&;VbNma(Ih>Ua>t=WRg?OZRfkKCNbEux7Zu8& z1nluWj7`5VtVm51$ta(r7s@EVnC=c`R_X=U(WS8dx)pg0w{g#JRvlzhe6`rJ6qt~? z2+Y=`BnIu&vTeLFtFWHrT!62%{PmGi@u-RcY@yy56si8;VyUwIeoU~)1aYMXLZ{lY zw(h!~GldZ1s)Nl|yw0!GUQK(Bd~5&2d8IB5?9~$QLG$fL=eo1-Xa`+cXs6Ct)ARs3 zX+gL@33ovY*k3$HwN^MYC20T!u`5qH=-aSgXZ*bBfv(TWBi*>+N$=e1&cghNF1LV@ zcsz?dJ3c4?H$Vr6A&(8{JSR_j@F{;^ztkfhgbn1ZZmcYa!HXK&_skMz2Y_92I}_0x)yD;<%e&FFzjec5-;NIMXpdcz5o>)>WVe0 z`;jMy2m=V4h`t*!x7I@fsov4<@jq-0KUMQDFD|^$O?(O?kRGAJW%shTG0x~AZGk$h zHDrwSHQjz{;zRL#6i?eXFIyWHM4BNiMy&-?$aGtqB>XcNX3~aO;95I@T06U2W4j=T zNes4CU~?Ud!+mE;x3jzXM9QCpB<{7e>Ig4Z@p%+&L;`dY@Gn-2`|)|9&D;Xjmk7XG z)*V?T6nyWq<>P?0QQ5$(teqRk+n_@93 z@kRGQjDB8UZ-<3_O`BWgz4QQX8;f%kF9Y3i!s@4$&NDc(9Am5-G^~UI0Mwku>r~*3 z*fQzf9fy6_buAreQsA`NLPz-Y5Cdd(x{WH~w_G53uxw3Fus@=`5lmwQe+Dg3${_`r z+8)s<*{Q35CF?sq0@3EnK$XFYTWckq0J+^4Co^RN-BDHq6a-D0&=poSx8;D#vrz;pCLpjDa*59Ed@ z0|Y_l<#PV`HtD{HDMpwiJL)yci#Z8NNe*_|YLGK-8 zpgG{;5i%d3xx95M!@$KPo9%k<*^a0hk|l`V+;3pL^zA@(8c3jqwZ&Uvz4ZQEK%=@v zuRzW~Kqx?_fYwC7g2RB;M0uOD-GM}a*2u-Z>##)1vQR|IuE60{L=Z??Qe9C*&O4X# zK7QoPgq=MdkX!JDDzJAq)>0UTe(~EW%0Q!#@rXnQqR>PF3Xg+C2HzHV??GW>E?NFy z;bS;{tq^*H;l5a2VvR3DT#e0t-kuhrY%z|~#R$F`jS#iu6b0aEr}8s9tl7Ug;b*vYQO*CiNk=+sD2%i z@wkRMM~|}0ZRy(W2g7!UoQA`QoS}Kc{CK6S!XV*y zXQ1&m@dw~KIB`%Q2p~8hO-X1VPKC#N{mb?hkNUcYJHg(wuge1`#@;_6>@&C!1`2%N z(q*?Hqkrr`!GfXzz>a7bdWx|f-?0K6$sg!Jc0xKj4l2HS@$_TgIs$@R^Z~U|6PIcx z9$5!XIv!Fwo^>XkX*2{y*a$0^sIyU|4ju|vfiE!@SDqRWf^8uvB4Rq?U6NNJTT*@ zw)r%rP6;7;oO@0!FI?T5Ce`*bNOp(M9aWosw;moVxi>ehEGIrwxwWw$^fB2zq*K|i z(e+wjP1xC3Z9)QSa`1m_wf-G169COYdB42e#Mx9@Pu6xSYu*3hZWt4;QHB{L$8vqF zlqTbbC%lHVG@)Oc#!;sq%z$dkaO?oJBuDLmxq}18a@2X-R>KR5s#ZOkN2eP! z+C}ACI3Bk(1Gn7FB;P|Np9Xt*>JK%X*{5QoRvNIv8iB3HF%lgwMd28Vg-7heAPBan z>?W|jrIQSaVO|o+iUZvW?1T0OQ`^D#pJh;>685ALOU?$i>X%Ms9blV zaxfIZL6J=zW^k+*70V&6`;R`lrSr3Wsi6o*mK}e$mPi?l-`A3Ym&#R0-;dBfj(nXg zDhhlms8X|DETc&x zH3we#4{A_L*|H&xGUcQwaxavZc$GvN4euZ>(DP}BXQOf!x~W|;gE|d8Qi5VXNmNSK z<7N_)4hd$s$@H<_>Jw!~y)E5pn6!zr2wQ+v4qP7Vcn5fv$@-4r{(lPu_#1Z$CHq(0 z$-Vp=wk5j?fFQu+_uc%L{c{=q7Oj_yuk4h~FA|5QVfvzLeKlT+ovYTnEED^!ZKGmw ze2B|Xgu{#yqI$Snvn9od2EDFOW3w(>cxK)R&BKxXJsC_Bl0n=UtCh;v*&FUSiu_>8 zzc>W&o94Xj_M`PT+KvpxX`v81Q@jGUA@@sd?Rui<{_JOtlKTYuMF&p=$DO{L`;tHU zRqOzQpwY{}c!HCg8n68w|Kb3oj4HJQUL;_;_n?m?YMZ9(<^6on+-Op9S{~%Pd>m{E z`KI}w_BOL*0sP!rynn+0u+<$_ZV9M7K)V5mb8Wz+$7N)Rw4E(P6e)q2!HT0;d3mw$ zQ;KfYKX)zx#2(z7-0a;XCj@Mu^#AzXIvF}WC;tNkF!7h)5x15kUxka( zgBoXS(LK}D*t|p|x_r>w?PHrU^xqOq7@zAr7kQ8RmR-NLDoP&~=C@#*eB9?Qx&=>7 z-~|7$oOdZ|_-owkx$ncQ*4!=jaSS3Dp))B`HqWLVu)LdzGW`>1!bjY0`j#yB1AqZg z@URuPU0TBb$+B#w6N}5gal^gSy_Na(yYK%W3qYY+Irl$U0Pmdt=PUpK;W=rI0~2zr zOgvBPoqe;us<-sKn}xh|5dai0@elI~cBPfQkoPURKA_W2EMOUWo$AxTU&s^#`C}%D z0lqqHiKN#lT#qDjA#=Wq%tlo-=n$4j{V&}Ugr1j*Dj{gsW0tAEIS2vYLCBgS(&pxK z<;K-nOn1I5PiBGC^%(Um)w7^+@IPWv%7y>fCL`+6q z-}Q?&C8o`oc(qC62!sjr|2G1_G2^0jh=Oe#==^>qhqqU7^6f)u{!F-6(Epl;as&RG zhmsrY;e-+x{wEH#kob?`4R_GI9Fs5Q=h-LtcCN9%m)2fFr`CONon8_9jp~1}2BT3( z{xQ6@|C`=9=0Eurq1S>?)VN0__Didu1yU=R)E*7qZG~0<8oE~caeezEep#Spm;dWY zXl$~=`36bH<@0X+T$30ghV}{bD`)DIQpRBqf|Bg(s-C$X`fXZ~o^v-EhOgA=14(N1p;*Ok$;ZWa(|8xfT4cFW&vAM83(` zJrEP830rI?IX?NQkP?BJ3m`8RLEUE=9(nj~DQq%=e&5NT{#GLybNYDr#o(O;Fy!s6 za!l0y=si45L4#k1p&R=S)@1<}kw#*O5$6rF+fG71(^+q%t^=lHlU`MjtxCs}Q?tT^ z^~!~{%5@*gp848v6h8JE6IJH+8ue9lYpm+2Ri3wW)ruUQ3)jRp_lgCT-dlCC3!n9I z4nOk3R!Z`)Xz6W7h(tEw>09;EdHh8TS@K474>sFI*l|i$s z%lh%<*Pa?d%7xwF@177u-|>g7kcV!*vrWXR0mm=A_E@IukykCI-2TGhzZo-~QMiH1mlBsvj>B~Im%P+oL2@yJa8j{~8$P+i0VOQY~uNW#U z*};_IPyL8hqE%>=wF7<8<;DdlD2*t~XGv0V9-U!8d5==A_d>3=amSh*zLBEinGb0% zB6-SCz_Dv0My@dZmL9=MEO<>Lg-}3?ui(fYVq*1&voo*>2dnE&crWilP79v>8#lgw zGIhvwJA1ZJt;~FC{S1^IPD4kJytz>_G5b5KvEu`m>=Pn(=s~04>NrO6m@$|x#1$QV zjMq10$s+A0-^kg8Cpn4<%=e*UE@yXmVVY&V4}QUBz&`lQVV4F_fhq+!}O!rDPVo%lAjuw&j`@3Y@cC|u}w%fFc z=r0qNPj`5hTvG&FfPf@t@iKFXl~aipD!d<1#jsRei7SS`G*K2M93MdB;VTkAXpKX7 zX%{L`uzG$lz$i^>!*Uxjb=cX3gu?f7nJ6W@4B+Md&=(O6m2#r+*0Q9VcO2h+I6W$$t4@j=b+gb7FF35 zO^(NIG8?lps)!egU9b8$R8+m+>aplm_i1Lpimz1HIvCc{=6Eq`eIG-SO~vKV;)e!W zr6x6}h_`cbpB%wCgx6%}VB*JRwYM+qfPI{z%bYGF%t4RIt%&4k&%=BLxjU-Sgl_m# zY)KC_Yt}tkv&17Gc{beSwr3svrA1qa%dAvtYOk$+U=+Jbxcgcntt&3ZB_Ekm>R~?O zq^HD&-(L=5XDu6k$f;rDuJ6TSdeE9_UZZ^8Sr_S^om``OsGI7wLTj2(uDd6qo6l4| zGb+OjFjz6kDF@sLsXE;Vbaor-j)rI`L`UOy-a;_YVyW@(FM@gLiHNZP%<|Jw=%3%# z1Cw%{mAC$U{d&6Cwp${Jh1yv$Y^erQz zS9d%&u3+~WjMYPY_E_4TMqcaFQ_w2wA9A)B%7ufUccVd_fh%~W&^1=E#8%3|#=@K{~$ zN^v)lHV9@uWY{%BsTZ;-`W4j;d}ul&)s3?notWb2R^6KlXnP5geCJuHkVJ(W)@9Wdif9RJ9}5Cm13;TdS#fR zi^O3bC1~c%r0Hol2b`}<(>-AN(B^^#PDhG*x(l8ZmEJ-~mlU~0mN`tMu7tYUVOQKG zZZiJ3nk?$e}Edi-rHJDvev$8a2B1>~11R&y_Hiv2D|`fU`bw=CNlmNzXH z6%ZeZU|b+UNn#)Yi*q1BUM3LO3UXahL~9|CbFvhub25G)o=gZZSE@9qbJcG)ko|Cy zeN9#euzjdeI8i-M+3g!JRcLz5c?pr9i3#dd{Ll_PSt0?2dN5aQvi`2UX z@hGu_xx__9gHn_Ov=fTl#By4*r-^haXGg)JDWRf?j>%qQ3PI(-Vj!ug%!P{(5HdbX zNp2uXn0VHC@W1Ly#ew5XV-G=i{H<++fR^b zN6Tt!q6F0>jHvvTW?c&Dz;gK;U8tQ8NM&N4kX4ISwI~kjaFj(LD zy~x3zKW~S$kojZQ0zs;@fSb`K&d>p-&=wtLtOb(PmX7}58&Ga{Tf6y!S_#pHIoT4) zj2VHcqdW}Qk)@R{0XmEnBm?8Cq;bb6l(33Fp_{tJAk^%26C(LWhKXL?=sibNg^2Qw z&Y^l2Kb$k6oI_14eRxucY@WB`us?Q8SeW>=*{@VI2oE@(3kccVOF;JG24Qt5DR$p# z&?K)6G<9vir7%DtJFNmE-zXb}ho?!4i27b2JGDJ%+}$A|IoJz!(h~3odvXpAu;>Yz z>(K4)xJ8Cmb2QOya5%>ZeR&+Hn3H8wMIaZuURB@97k;!>8gLAv8+%N^D;yUtD}#57 zRipT%s^utzbY!LwL#S3YD+SoR@ZM|zRs)9N#Z&BS6W^`4xKp>m7CKl!@L9z9MfV)v zP#8I^lc26R9|;_BU4APkvWWOJMdQ%dQVTmf%5HZsRcRAyNRK|4Wl3M#t6PkjlnMI^ z14p6pE(lkuJT2M;S2)ure<8PO~zZMxf1Yg>4J7D_;fFcTe}4q+j-xB4*Q!OhUmjO`n52u#=_oUUju zc0CUGBD*uoV@1r1tsur$ccEn+)>^j6v&mJZjoEtygUU>yD^MzzeYjaHXYMO$w3MDL z6-$7>5a%QxW#3F~Ok~-Xg>q-*svk_FxGkd>x8%AUXN~NF>JQf3Qh5n3Cr9yh6{TTrUr{oMIv~XyvQG#A&yS|WIJ0}L08EpB*=I)fg zq18V9Nad0F+!W+yafkOnkc;B#yIr3UxroihYVS?7L&a&0>SlpjK>-m#e5y|duk=(c zv6*B~2y%5RkAvgM5Y^EW*vd1w$)X2@s6e-S=@!`zjHu9%ba;K89DObWkJz|>BnYF0 zD>*bEw3RvqzTzo8gg9u7GdlcBB?`Db7~}@D9-C;Ns@-(SIWVbA^j)y3G$@;w2iqzz zwM_M0kQTzyE0UfH9youG!(m%p`wVh^)nfjo+pF|fQlB{1l$#cc=mK#16o1wfq9OZS z)Is{xy3cvw;(ioi_~JWgmAE$(kv!P0D3N^hE72U!ZCR)~dL}qf?dbXNziWLJ2kT z%2rmj7?Fe0Q9>!FKc!PrJFE+&xA@!zlTLq1rKAQ1diVJON$>kw<%`-Qafz6;fdyt4 zNaBoQyNgW&RgHop*eZ-+(?VV_ z(z8T$n=QyY8(4vx0+;qYEH`T9agm@IesP z6(|EvL;*wtx9xmr{$#VH?~z$^_}XZFlQj>wV4GbZ@ImupChLZA!3O(ND!YpP&yMs& z(c22K28toiLrn6f0g)i18&Lv!8b4wq|J%Yk%Kb zU(%v~J+-L0D#?-(ioi)ON5@kvOic_$X>xX=R1#^TSNh2`X1gsxV@J_pw_`8h;=$=jQ z(%nnh%lo|X&I-=kSJ_bETyUJ*9Ye0MvRkv;?|h$>%-$}Ljl#T&sGNQa2q-$9X1D?a zuEn_~>(D14K};zQWWfmr1V#GAa_(drQwm^8NJWJFY+00iUG1u2MfprcNKJ&g4T!o8 zXa0)03m|cb(n7wSs8ZPUA zO*k+Mn8U#s-qo#=TX5#Z1M-;b)DQG{75EXxGSpvSdwF2(n1AmW z_>X1pETFXnXe@-L~Havz!D(}Zn-bZ0w0Zm10D#yUgZ6OchV(U;dg-kdS5j5 z9u44F-(9;IulJ|}?ptnn`mC^!cnF+$j;*RFj-J}KBb>VA?M~+}6gUEmg3+-o`M$d` zy5E~9jkw--7#IX7fiIUvb7~UP0D=kErR9E&%k7iNyz0oX-MUr4-B3G=)LRzArKSzw zIqatk=KB%LTdjHX+O{v#s({n=-k5juhR}BYL0GHD0~n+lXzni9choWVF#PQ2o3v$z zxNR`^=c#~n5=<({K@!0EGDcRqs#`}CC4J<;A@1SK-&HL-wR{5kO`pFhiy7z+d;yaX zn!hP6$RVb_0hF%^=8^SC8|al}hk$$6oz>PXkkj=@CDM)e401UI1#+2!-+QV{0$1Lm z+8$BvaROKu_0P8Q`jE%$XQ`KJW-eT6>3Nw{>5SX+Y=s?+?>JZ z$EWRpudauAh3YKa`?vtzjKdTSEBiA;=?I%@=S`%&Nmn^ib$)qan+$#NT5^4vYBW2= zAILf@zC$lLI0(1o&*12|N(1Y7I_+*TJ1#ppihu=0N!^i&sDp#LD>Q939JU2}LjOm9`Y>~FaT<6Wz4Oa6v0 z3g4&yxTdck)FsJ4fvx|W1M0j~^;7Wt5a(RnyWX>TeBdJ%>WQ#&N>w|N8!oBMDtD~Y zxl<2I_GC_T0f*K+kM5AC4r{FBHZq8dIpG!@ zt&td2XhA%-T5UooTu$r^ZpPq#)KC&)U$+ZuPYBb4A@s-2>H$}VZuFZ)&N`<41i5d{ zj15=w;}g$$^KMg5T7+Bg7mPoQN^d_HqdW67P}E(|Kc#~&KTgFUqk5xGS*G~XglP+G zd9Sg{5NagAWwN>2MB%ltSbfsYM)kDZpWKa4zIltbC~$r=fjC|*Kmk!en5<%oa(Y_H z;mnxR^`7rKxQ&l)ktJm^MKNwKP@6(dp=w#ELoh_Un!kJeD-~^p+T$=0dICn7 znyyZD($%lQrK&us!q`Z0{6l42`X?5f%J>h*v0?DXFLk9qo6SX;S(9iFvK71ZwW> zE4Ls`wT1Ue;dQD=r-EjX*MuoBBIK48tg7EH>rl{|e#cxYX%N1h6WnSaV1yD#Z3XUD zuo&-dho9XXN!#a)+c>SjP`NvxVyaT+()+-%d$`aEM=Ucx%y<_jl6#3s%Z~J_wv@`~ zYcsjVN%JLaXri!%fY3VJnTi0OM9-6y29~OcwCl<*y6bZ)icCJpDeV+Fh%t%S@gJOA#?knd_NP zE;Qyd_42yH=#OeZvprQ+N}9rI!se&hq|d3E|AKddx?rww+*C%NYY5xIr$~I(nxnwZzf%wv9vabj=v%X^6tbBJ7}KK492wepmfKh zRpcR+&($w{e!&`{kXX->CzM9&EA18PfCw((Zi%fMop{7e!^xqdz8YG37%kheLl6~Q zF-&}L@gpBni%>pg!|2=N2@+&~U{bxXgwpB3yJIourBarmvB_l$UfM#kqscQQc+!%Qbk%eP770&coRg_&d_xBm^jmc6XTxq!%V+wh02XoB*)Fljj z>+^H5$0m<@rxe)aF}GG8v^`A)?@Rk08-$|@G9Hm_@wt3w=6T#2N}tQ-=fLB>-K^wu zqhN0kT@mJ@SHZl}Omjo&r+LWoLinzBS+NSx-l*mtg$S=)t_;V|O*!;&8otBxg`J!u zM+5#(yCIcw^nW+#VZJJ>A1rF*)(j^aDIHLHH%Ldjmi!J887sd&cy(HK$f5Jc6>vblSLfY+gDt8h%~5cbt4Zz5ejOI@fY= zWWsv%{&DGeU9~PisH@8>vT4@|v|Y5~x8!%*)PT4gPpWu5y(}Ss6xEuJ0uN3K{6$IvyP}6{cs#3cxfdgr5wZ)BtiSRlZVP_tol@-&3+-yZf*Hw(j5K%b z=Zi3A@(I-IrEDOL)DapV`F)S&Gu}exaGYErmwd@f3-Yk^(9sjS6`i>K@>}h;HE5xjRV z-Q=pfS%_Nfd(hNBErV#slZ!A|GD@c%@r^Gg^3Bj-TE#G^&b4=WIcJXg#h9DT;ZH^_wpGl z{fTR+Of*VNMwj!Mgq*wA!0k8f2eR;DGThGq&midls~evf3+f04ZKUjKMl;e(`I2Ji z@dXl96ucv_S|>FH5Jna_KRca;CV1$XwCEXHY=_tcLkKL&8owUiG>HB|ZDfknt5HQV zpx_9>bWK_aT;_dP{uD+g1*~e#UW?rs$sq)Fw@l1EX*SaoFQI`R|7}9uTn*{U*iNk> zUFw~@%6(Wz?VgtGoSs|o??Ma&2tSt=LhY-(;q6U<)e`%ZljA~m<7AfCkVMQtoA;2A zopqwV?-dNsxG*Ex#T@YQBYq-hLzQ^~%B;%007VS=6%k{#iLF0}$U}V9&rGWx(P=z* zI=EsHB^pj7Rj1FS+XM4|x_&mJtF!5Ob#u+>$x1aj^l%F8K@K{{oPCWMt;e^L`N{gtU^k&YBf)Qi@I?4kIhfxXE>!l79J zgl&Tu^>wdwEoLQ|vC$ zUaZYsy?_fR+qXmDF?~c;Vb+fhj=kbxSFY!tDAAXA!w>@1kAN5&On~FU00)eZN z45=pU4p;S{>VIoI^*h8moRgtIShy;%)?OHT2wg)ODip_{5(ld?T|))gWdoh&PXZ-< z)@YPy8=!cm12%3;^}3k0Fkxl_#j#!Sut7w1J4-W|0tAzI1*8 zeQ4_Hmjwf8O(E*?zM{qaIUMni#omq)1k1jnrhH{t0_&YkNW=KaGzf?>2(<~0=2qOw zgQ(&HZGVl$+c>fW-bCKU|0;nbPT&&-<{7>sLK$;5K9`b_3Lv4(P70yvR9*T%M>;so3V5s-M>@mGP;a=AV$iR z?z{9wGvc(OA~upydew3n5aJFy?Ajdoq%OJ^Cz@B4BK}OI42EINBlY{fb`IV3xG;oj zyVG1|BRXWSGq7cAYoc+tR_dUrmmmMQP&cw-iAs}>V`W)?Jbe_7MIaO z5BQ5E{ID)Bj&1+$Et9CJqk5oGvxyy(Rk6LCt<@jXKGJW7j%SRm3mKj`os!eNuBG4^ zZOg$UENBzm4i61@t}snQ4$}5%u{B%F(+RR8yLU1Dnn`&2)JF{wVRL8F3VCA+GHu_J zT~1bW-|w|F=;=%jR9@$P@zGtV@|$m$*e5E@P3F3f-0I+tJIMKud<4BW(h4a4k1|M? z?r#~?HT$nJh~sY=gzPXJf1HPWDEOac5b4s{cNc+m9HEg~yD6Dh%pnUrRq3Afq0^`c z_T1*lWJXO^eNM{|{~N033PSZhLn;v2EKnCdS0JCbn&BVoYpvf{E>k zZQGh;qSJYv_uczBd++mAeW&VFS30P!^qqfIy6<(b^;?%d?Xx{RH|XuDx6d6|U3e7r zv;tFbUKhR3cGorG*Cd_|)so$v=LeM%R=+c^epQ<&l&B`v)w6!A7E@l~oTgPnBI38a zApPj;eTa)JxA~}BX6(DRjxBwy8s5$$xA|MA8L#P_JNu#j6aiv_7?nEovK{#Jqm+pk z!P(9NTwF~Ku@g9Lai2wNn!Y27%IR0DW9>%Q7c z7pq9P_0|p^7{X-1uMf*At86|pC5Tzr%;PCMeGEV-oxIT4X#FG{HJ#FqS-g4zuA*5& z-%Y`)=XLW|D7vI0iVCIGGa{H!;@s+NtIqaFg;I~#enF)4HHcs&ET~&ao|qOJWpRe` zlQs@d(|S<=DR}$xE$XW32)qN_Gcke6m#I35ao!zBFi2NurME-VExeU=HT8JD>G~Hx z{U*lM)GC>u&c~$>y5N&>wFObG>POXX&K24#=&emv5dyE!yb=_7Tte9D5}yV%X-h|^ zmSgvVPOlCsi*`>ivWWI+YwnXva*aVm+yQpSQO%T{hD=-j**Qd}Bdut6y?l~dFkA`CIrruzrR9Z(0+I<)XY(pJ1i6Lvr*b;wq`XhJ_kvy{ z@Fmvp2|;c&zoyq4WzyAs#CbGKvI6O4G-JP6+jYCjeS6YP(V@J3$!1ezdY= z5yi^X=_M?UpOQt_xSL&dz9#zb!4>JRVz}6R?yaHPjh7YwVeFj|KpSbb$vAlxP|~Ev z*I2a5iO#YNsz{GCj}g`LAompLvfMeMo9wEnX>(dU-(+NYS1c_*;S??qS%)Kt{Z6*2 z-^q2_ODRQAJ2_wdz#8|*{Ylt^;PqOVTP(H;;t)^Un3rq>L(IM2-1#S@O$yq&lN>OD zP%k4*GXzD;rCJt9@0YCkHX^U68X{#~Sj_VVWqRGi^P~_#Z_tcWKg!i62t!;RDuiEu zqTmx820yPRN7Ci{YDN9@rf2%C55e*qDgyWR-@6-1Z{Y4G^9s1TSuHtt&}jW+!^uvk zo2PwYRQ^dsfkz)MaONrS`G=PlYOhUeQ7gFD9v8lrGUZ12gwA)RAal~NmR9`GCGey# z)!K=ZvFA!V^<*ePQ-7IsO>^r&lMavou|mL|HlgRMz_W818`;Ra-k=V;mfm}j`Dx?% zupmB)I)X)CFxw=^vsC#8^5O31DdG20D<&uW?Z|vs6{Zcc?V|ivo_SU4%G!ha-bhIo zi|>jhUuX?;Yaa_%FEFFA;va*;v8W)^twWg{cjI&2*5WZ(O+>h;-Y=hi{ z5H4um8Rm4 zVg=ZGn$c#njm1UEpVjp?@>=vI`7Yqu|9DZr^YP_`jzQw?$My_sNw(`Kfx00e>rMu^ zr!c{3Nql=4!depl>#-0`(~PYq)@+85pc=B(kDc1lt&s?GVE z)bjyAdS;`b$_%F|rsCxBbcqS|$6hUm`+7Yu^_nKK3KmDO?)L)P4R2KEz1pToo80eU z5EE?WzZUA_KEuas`TOgFI~5sX_CO|oV&lldKLLv@%o8x6ya$O}6Jc5hNP}6b;wX$_ zF$2w*6+kIG(2SV~_wM;j%o59|DW>5rMAvqz-M2<}&FtOV4i=uE4u2@l%#e64%ir3( zuDzZS%O@hS3nkqi!<@nw!{ib%B+7Z6bS3ET{xjQQyA_$yDcKHTnZcehNF4C$m>(At zy5VQk+HJ)?eeJi=NslA`Km&=`&kG+}-XNug8q|%X%9s#Z`e&iwL@Eq1z{P_S9=OSAd}68N z(BSO>LPGJto%Y1P?C?_f}x(CVr3geUf(f8 z#y15CV*YbGKhZ_kn6GKV2vc;w-1OS z&Ar9Rb$S{AVJgN^g^b1oV1Y1wZT$c%6vNLz3{0ZLi@$&{W3YkuWBu0k>wo-G$|f|( zC+r_x4@P7h!6Y=ObW9;KSk&-KYl{BnrZ%H(XzK;S-wALCJnYW|p(R` zaOFB#d=@%%*u0lGv|}P4&L*m>uE<-|`3X``0s&Gm7E5pJ48_D_aK~fd3|tR4a6RI& zss4B%+`#n!(*+;vag0uJ0pY=52S394y^eQ;#c-u06z37>Phbj_JA`5=6jwAOCmJVX z_Pwze_s^@hKH_P+SM)*@IiMO0XiP99g12O2R50sIGTd2Kx&l{p?y&hw_gma z$N*PEhYef}9knJ1Bo{R%I2Sc9;7iIUuuNdMf+SOiXZ0v~3p59jw)a=ib%b1?wwO={ z&l)3EBFF`)|1;osQ3BxitPyYMGtG32@g(Q;V*rSeb7s#Qs@)z+Wl=*rL}m@l7Q16f z1zGpr5Pr;>%a=D_>x`4778~AkyG{)^iVsf$-wOD(dyn5m;Jf$N_xW)(phH3NOo`yj z7k{ze@`fY~CIV&`cv-(|(4A*LsIGm@J$ ztjL;_Yj=}jQ7@7QO|q@go|LacxPKvzYMgbZun8CmZaM=|JxDw}@yxej@uJ*87FG{d zz<#0#?mbkJ7y5E3#vDLy+#?2QWodD0ur1x{d3M>V%N{EjAW@A4&XvCrAWGPa`;4=p z!LwS~pV2($dYzO@=qF6?T;Me~KjdqV2;8m0H!3hKe|`^nl%)NRBGf5&n=f}d*^Diu z0+X)l__l-ZtB@4gc z{|pIM%ifMwu${Xw74jA?S zaEwRa19cvX5V~z2R&1+#Yzr}?%qqOeyOC)@!Yo;n7b+3@Ol&jESZeU>Fd>y&EioE_ zlTFN`o)GB79n0E@DKc{cwm*#aZEwU-(w{0OT+_XH8qyzczL9^u`KmO*nskdh4xpa= zcN>JRE$i##1^((BPojcmf=~!W>%QVR#@krxj93u+UWM9E)>K)In^+}_I%A0+<^!I8 zOa|SiW^1Lu?taXWe5Sv@1H4+y&%AZ`AwyZ$CaZB;TGx6>w?eOESfg546@`RoS)Tej z^#`_6tcsp!QJ3q>xbSNnwrY_dZgjRB5WkbWfkczfLPmWT*NGEUPcTSJ-3RxvSn80| zQOI*|J!4*yooM_-t~WL(cp8zPvWHI1j1Q)~p26T@F)+nXb1&*>ER}?dk(8};lfFD< zwa5XTh1_NC5cm9_o`%;yJq=`w4^LzNzj_+ri}s@%UzjRjDU_c&MI1(wycyES#0%Va z0#d-qFWC=Ijr$(uysNx98!}>5W}@fs4CiOI6#@>N3gs2P^XDyQx4H+?uRS=&lrGs= zEq#x%IkVJTy@M}@Eu6{5dBDQ8F@m@ygQv$>f5JoVBLzx(sP`X%iQMWK(L9?tMqH?nT-{sC8P*t5GIKQzuy>W_aZn_jPJV=QLK5m zie4xAJ@|zoezQ`@Bh@?Z0IwQt!)O}SH;Vms8SnOjvG1MO1W-)un$_^)avy6ng9d?j-vGMg&zCtr!cwy!k zQpmq`e!AU1bpCGkzjS{1zjZ!a&i}6S$LpvtXBy1b&y){oM&DPW&p)S6rrJk|Q!H^F z$$@KcKzk&Sq(*}gzDEZ_mj+ea%wAIq+kJA|>-o7zeOdY#EiHwe-E|b-^9^qJrzi7Z z5W|HY98_P!m=nuStlyIfau@lfQ2L%4c)t$m1HNU?@7UKBO@yzq*vOPLRcOz~e#$|r zA=R^?3=OQ(YvVjx>%7vkeazST-CDH#TaOkz9C3+FY4OOlZGGFI`)%{wmGzwvZgK)@ z(laQkr*;w7IV{L*{cT`ic?%7Qa2wc4c%5%)r@LGxwdHyc^PkGMrFL^#; zRaWv17be2$7&Vw49)I;Bp%0dn=^ygkHI(vbiSz*90ZbxHH!$hl1AED_ZS{mEfl@2w z{c`NV^ts-JI+fvmaMScc?PyLuK7))zT4HHhZ1R7U>1j^-K-a=(ctYiCjZHuiIgZRb5D>MN^~ z7EBkcwI+utEns6_@rv&+@1pFee^V&`hj$TPv#qt{Q@Zgj6_$n{0v~E( zJ1q@CxAAEN`cL57osIO4HsX-Z1Ep&wP3sJXJ~}-{-U=Q$ch7#zzm!)cC3=F75r98tqy152(w>GQ|&bhIB&F3TtuMQ$FpLomSSKW1_XIU6Q$*6(p38p zW@b|^iHNO!!&NzU!Dct_2mj-nUHp*A_yt0)#5|pB_sm6}boAHJAy!36UTf~;S(f-% zE^#_hg~ZO}tYC~e<6&m8(jD#qQU9j(G0RJ{t=RhkY@9VZgNZv@S8&wD+2nwQW`?7x#w{t@~`eF%Ff|CJk&FNx3l3Z3NLu5STJwBrWw6kt<)J^Tl`KeU2hh{OPB(7 z390g#QKK~*lVYLYXFk&hUl#LK$Sw6qy1XCqre6UE zX_Jw}i-LPTGD?ZTsm49Q#Ajdo5_D0=!y^rT4^|A5dE(DI--!&UcT0R_{E{=HdSm<3 z!W)iftq7G1jItJ)a_i|U9(Hu5P|_SWLee;6X_E{e#~X5-v~J%HxQoJ65%BM!Ib(P< zHhKCwHIjVzomhj>V{TuNvE|4s}LX{odAc zuNp2p9(%+1(^f6lujRXO-iY#GCHX6Al7ndOq-x38hl_)VM_lR3S%-Nu16!ls1ngRL z`8iJr-966}f2y`#E1RTI`NlH5la|ad)@Di95}C#x$L2N}9h#kgdNr$@N~zY&RMMFLi6*Q4tq~%fxku%HT4tDzGd8U5qX?>^KK=9J~Cy3He z@8SB%q+K)4_jb;X0ZFp2^U+QIeMfKz-R-;kjro$tOOV@-iLH8G0_SJ5z6~)y{~>;x zT!e|RzPy$1TVvPu35nVSzd+x8!`9XKaJpop$KTOk&=#Su+nX&RCiw2}%p511=zQ^I zi8ll^(EP-s25z6iF_{~wpJXu|>TYkRCCNjPlldkObCmQs6hWTcWAuJEN881*1|P0# z4zFkZY>2xLM#`+G#N8M+fF?)3;#${18O5=?Q+*I`e$5aQDzCmg{9{2_&xF=IxY};9 zXSTKgtf-@YX<>=}aKv6run2EM#1M(W`vgH&YCu+o^eZ+1`rrgX@0U0`mo|G2%gEf- zzG2kZ>#G-4wsH~#d21jX)IYEebisee#OmKlh`evd0jJM+&XlGVh^sUm{W$E5@N4|T zOR=5m+4fro$+HsAQ?@nu>i~sXxPW-9$jzP?VNm7Gm0Xle6){TsD4hAiYzcK`FqCRir>iD(b&&d{#PPl6C>*{* zBYDvIp}&Lht__t?!MVkfnS{tIBq?Q0 zn*GMl!qdQoE*iH!PP>=GF;I_>N(V_pCyAD_BdU)E0 z5#}?@OQ?E!I$t1s26H5t6hj|1j-M1WztgZDwJ+u+{hQ{WnZ32KF>qw+gZ;vlzYo zzxmdXl*k4K#7&BRc48>nXgkHML$$N~* z0A;K?aB?MPkemm0#2`R_GSs48V;uA1H;t8J3ZOf9=RMVjYqfamYOw8E1$*nV!K1x? z7t_bW0tA2{VFA<)0J1vpTz~+uTK7kAjBfm|UUUAJc6N#U9GUlOwK9M>k?Sl_n!_-z zCP*=}FjQk{pL7Z`e*mpv;*DVLU8PhUe36-gCuEVCY8TfkbR+qRw%PkYs;71)${3P9nv45PJFiEDcLzUdmH zRYa-fwZ3w&xjSgN%;9&sT5Y$Q33CQK7{Xr*5W~$-g;EDoy6uRG7GMvnLC%|$g^+d1 z&w)~XTd2gMsx9H9hJ<0aexfV%SZ_5!@%$nFpjRK#XLdFnEU_!|pUv=HvWL=~6 zKB1uW$$O~Tt8eRDjoF?M{M%RJDzsXs9@^qao{~SPRlk2?1Pg=LRaX3+qY=*usTU`xJ=m3L-6B;21 z=H$$RUkZOTH3llx9%W&2LQ&?t1Voyr+<)EWe-A@iuN?Aje;csVj?uk1E3mGRtagnb zp+pO@VW(Rr`c=Bx7M$M}Bzg-E&41(fg4Asgx*|s9m=^duRG_i*e&gC&_3a1T&8?q! zc+Pt|)Z_XV_GY~nU0xmZz_oMCOIV+0-Wx6qbR{*~w}p&*C`gkAiq#_0826L9!|$4m z9%n|JUAT_?E2-1 zQ;zW0*^IX3pOQ|gFa13Pl}mnnt=ZP~)(^4UnV*GB$a%675_I*YCX{Hsj{wxfkDW*8 z+I(#v_QqVILbt5XqrkPS=X`PGo=m=LbgVA=)jl<=6N$=EEwb+Q>vThZHKTB+BRC1p z^+|WJF7}V#Y6rV@rt*uv+WRIU&?UT;eSNo|!UrB*a>QERzwf}BMZW8LZyu2U?vz;T zHbjm3T5)Lu+*7+Y+~4EB7)3KO+F!O;=w-%@dHHkuV!~Lmq<$M}VjU>>>8xYAo^r51U(( zsGwxN27-?6u^WU%U;Luqv@!?r2VOwJA<%a8MT`Lb2A_AlBS`=*sDec!ph!^Q5)scv z!t;AEKC&-(a?8j$pp?Fb)}Ti|Cmu71zxH;%xj(F2(?=3Mh)g9HYO>X46xymL$yNrcc(4)pRFY6*$kMEf zVma>j#?83Yfhs$ijWlAb6q=zxCGrbkuuWFp<;p&o0a-LHg|g%#wTP(?7OgKvJ>{E* z)+mrgGxY?rXjQW4tAhFsA|1Z*aQeS!ltboN)Q_VgFuEPRJ>1cFQ5Vqlc}3{rwu8_! z>#bv=Z)mE@RW**|me1*0{wxkR?a<2?S;{os52>o1+zPwe9JxKC2qX-8Zy1X3h!E6(D981Qv?qrw!WMXkcVc9arRq)XX>k0+uYpPqhYJk83M1MsnxqOn2JRPQ~ zqG%G?FRE{`6#3++f%ZQ2GgE_o{iY}JT5F;s^S>nELk$IugxT~JDw9(99e%`W39t9l z(4vf04%GAY>K84ZpvIZADCR~|g&5UKOjn@8_R14N7%h^rgwbFL=+l;2-cE}+P6u-* z93+lK>fSz?_6XriQFLn8ol;+%GU=JCnoVq4T$^B{KRX>Wj@_W-;G5#@tz9@Gh}7i{ zOAn(;?@{k*_^}J-jGG9JTBki!V&4aa3ssSTnt}w7umM0oK7CSw`-FCk4Gx|`h(zVJ zFi!$%SamjA5}2<66VbYS136MoO~&}0=8}x2sN8oQNj*Gr;~#w0{#VsQWkX>tY$Kro zOWgbBlu6Xo4%I07D|;b`L!5|`QE>GSyh?@KuyL?*4vCt!++4?3S6Ge~lB~C2loMW6 zkj{h7u7?)!Gh4l!c0|-zkB%eIDbvvl_z8?vvt!<&V*#zP0IYGBTR4gl&RV$!Xl}Z0 zlaI&zbTBBPTcniMk-bY@-~?U;Ri&?cN-H*M z$hJ;vQ^S=Js18zdBUz(lAq;ZT1^#Xma7QrjH_$%1-%2n5DFczGCF?5Gs8PnG5|u3d zxpC=B`M`}hL@9=qZKDRY-?(NxASp`tr$!yt)zBX7h4N1gr$^%*|G*_Dil5L3r0M$D zF%r6O`}yalob*2^~28-0c zQGHOpwwfRTGd}LD8}k1Q-RL8T4}&zk7gbyV=Kq;O9f!*VM}=cyiH4F2mo_1^9+)DH z0f~_f09&1{ugeF;ro#QKz@bJ5X=?lwsKAlJ2ybc(F~s9`zWW8agP3Oe@{?x>eatG| zUddhrq2z)onANZz_v)Au0{vlD6#e8Jbs9q2m!PPTCJUaw?5p6Q*ib!V0w}P)5mZ5> zJ_{qUWdt}%JSVCpgrzS*r=zZhyvxE+>n*MTxgLK<;05!~(D#`nLr)S`(HEv6=sVJ8z1gq8$pN% zD51Q7hCb=4h)h{y{g0J(lleFOfgdYeWe>RxmR6eQn&wO@-I%plwOCXrx~Aa{TuV@i zAc+Aa!;wL7OTgQlyntoew7V>}ER<}?&*VT}9NTW!ZdX?$-sA;8gtk!*S-jVD+)Kmr zwom&_3bxwKCkj-}^a-Jkt5vD1YWT#t7fnB1n#J6@VeG=hKo#6!Lz)MS47Y9KvxT>9 zs$lJ+_=GgvwDX}WlGjBpK>KM5^;wZCrsP<5I27rP>Ouki-ftEH?>{uSng-wAE$T|y zXuqz%L>;23WhD?en;0dNTy;cBhx-;YxWeRM@4(d?k2!>@e<5E~=2U=F3?(_H5Xaub zglhk8se*l}2iSmU|9%X5#jvGIT#x9EsT2otHHrhW_3P5Oz~7DhfrGtr&6lm7_bAw)X03pgTG@rL+{Cfc* zK=2U_l8R3B0R*Oxh1bAPleD|{$>$6YmQrH?1Oe!mB8(yU_mTGk!hp`xr?6BY=LO&C zP3J%0;X(O~6Hk@m{|!8tjEIalG5DLPmh~brbgn{Z8hAgM`{sR|6rT~I)+d3zO?+^| z{0&YdZ+oIgKu;JCI3e(=>;bO|{u4$iG?6~|Fy^-!Akl57RXH(?=_&$pL~92=#n}~C zC8R$hzTxjX8|d!~bY3@s&TH3)^V;=sY3>OAc3#h(@ocEDjEPtLiB=!#k3Ikm2VI|k@{@?&Ncka;UFBcb6nr75SX55g)H5kQs%cn zFXs6l#0bG(#7M`tH5C;1CZZN$U|Lf6UuntScJ>1^)}0Db%_MB7W^P|H7WsWh<>(EJ z5b9nBQEGT|swPx()1IqutbU4bcM;FsYc9*HiaIRN2hva3AJz)^oAAgf30}{CjZ5;T z|3_RBvP1*|W@-VvaT(R~yex`xsn#NypwOUD=GVi3okX0*azZCDe9v_|1Yy13i zrW4TrF+~vvPkT61|4mQb^N*~rRt9@srp)BSUdd#uqs8$=j1Mqcnew1c*l?-DIYMR8 zF5hPSR0Y4^OwqObCgiJygWdj5rsQ-17^Xx^nEE?RNrLhbrlf}+ME>O`?jSNJyx4U7 zn3T&I{QSJN=9iLHy3?S82k|=;%$xZ*J4^Qq1LH`PO!AhK3ttOSsxix%F4^}rF-5sf z#}=-PNo^9)8*8VqD0&z1wG<0&o<`mE3|Gg_D7+VD7q_2Oo2@xr8U{$#|2r+EP17<44uXS4xEW zCTq%;&~sNT@7VtzV9C6-Sh>evU`guG|6{O(j z&(o;RXUMm)Edg=;48o)D?Qof5|ASfMvbQOwxJP<+l)sMNdWye(lHwXtZDW-$@7FsK zFGYoLaHcLnl`UV7<@$d>OYrsyq`KU_c5orThZG!AN>Vp+K==sH-P5DPPc8|%+*S#u zALYc)C_QEw0KpRNi`(kh11Hh*%{mJe=n+b-d`8J;HX}v0t&aM#7PkR$vOM}pTIZ3B zYl1*m+D5~gM3lwei_iC=vT)QN?u?qBtZZ2I%F7@(9dDsOBHuYkyb zkM6bUw9zyQrhs&D!3gfPIzF3>U$BDm7}UC8yU=#mkH@MZ<*0rr291GAre+c#S(4uQ zL6+2qjyLFUh-vae1IC%IKb#j1vXRt!ve~^}Yq(_=R`-F%%T&q^g{k#gwg(cBVWTj* z2P5WTqe)C}OAN1Urt%cB!8L+7k!7?349AfiU#6m(1cqYB3l{U2#mncQvD;G`8S?O&g7U(JUqE1VK|6a)=Lxi@8xt`C9apTnvfR5Drrbm}ECd8yX(iIjD;fr?wXCKR z@K+Wy5OiTMxCz{?kEta$jX@=7`Lf6q+wc3GtIcE`eH9 z&!}Fd87f75%Z(D2HAY}UL-UcRiW_0RmuI&uXCWi>K* zc2n&dYTBFI?vs@@G5>mYz?O3eyT3JNoCX-x466oMDSM(^(IMx&q}lgRnTu1<+jSi6Z2JvOk0g|`w2oEsX=qCzRKsl@w3gk~dyJiC zE4x^YO_k@7K9<%1butb@C3!9gx|J=E#O?ehuCl%-HL6;?0vY#7<*Fv0YckW*bZcH) z++6dggG--KhL1k=3J%-bJiGJ>J;`-&%1wV(m#?;zGVS2snz3>7=9h!5s$F@A-7&OK zJ-t&I%6Yifd0v%DWG{YbRXeD5ydIq?t(}@2Ex@UaXhYPgEQuWSp!XMq4+LZ)tuGOz z(=1;QvVX?w&%cT~(q;Zt)B)_}FliDTv^NewE*1iYIro7?#~6b+mXw8qZ9yML+KE&h zZxd#fvKm^pvhM3&d@V#%YQ#X(UwmyB5sMyI5M_10*Op*CJfWo3qGf}&cz^}VMp~GSb{Z+as zxv#3E|MT+X#F+1cfSx2W75^wuII0h>{nK z^HyFqJLRtPRKB+XQCpju{8PopW)&pMSyQOF;#8Q=HlAf{bH$g19Q<}F{wsF}3&p7@ zbJ^(7LJmA=C74kl!Zw{mNOKrtV1m?0bAg~}ctQ%#;av104ph@*#peRGN11@xP_p_)P;%9;kh^h}zxZ9_NRfpc!O8H6Wo)|MC#iRB>kp@;pEn~xH@ z!(pRay2HP7Jy2IdMYk}a(ERn2pIHD>+XsNSjzRygCJRVXhB@qW?W}16O(6GoQ0ptU zPbu`$s1BoVRj!LgV`<0TNOT2p_#IWlp0oRx*0kQ#Cmu%iMaDc1jmB7A zV2Z(tXSZ74Y%(TrWufaSw{r{f7So1$y6ta(*TEh~W8?Cy+rF1MEb!d_obBg$7i17Z ztHU`UBM@XC2+D!5*Y0#t1R*XdlmRoo3aH$GCiBk<&-yf-w<~_4HhU^2tTt#+Lh~6W zrSE1?bMJjrpYGq6!A{=u-nw$dP*30E_&50$bY({zGHq;;aj1kkPCW8}#JSf8EPvowFM8P%y_+2?p)v>E_|E!EfAS zFJ*J);**3t069=xGJn@VDu*X9AT%Hh(+uyLxyrj{$6!KbO6(PXR`2dwVFGmp-sbm` z|16oad!!M)vU1M#tsLTyj^0Ga=o`rsG`?j;-u)n#*j6V@&Vcf zB;T~ZOP->xs+S3KFruVUjTaWX+#Pl2HIId%DAl%kmnyo0kIYVtO0Lq0=ZJWl@PG|2 zDq|uw*bJdUpW~fT!Qg?NUVC!vQ!eI}4r#K3{X-Kw$w?1nj1tQji1tOC7B6(8*6Qozd zz!YtNnRqhsHMx+He?KrBi~>f6LGlcUFdV>Vx^^V(?vLs2hyLQ@KVTv3KTr>5;1I#o zy#ovv_wIazixX?Oh`J*?+7o@?<yg_$Rk#fFCiFlLHH=RC$*AjpB2L(k_+`LCr z;4`B9`bp$N+!B9?+hBX5PawkKJmADYaeMV4ZeM`n)?Xid3$q(2Zo5Ck?H1+(56B&@ z75Gsz!jXD?0Y6~t2NA$v83X{-lz6z z1{x&nhej3{Lahf`KRyhO@LC>%L)ck4Z|k-iOwWhbCj@GJLZH@v{LuPaA6h?DIi@^l zn-$Op2nA|=EJ&sgtq<3SaqS-(g#&_94Th6a*$YOL0t-r%f)5Cn2?jw*mF7f7V$8C+ zv-A#_WYkO+OyKMK4-~NE%_5GH^cGR>yMAMRrEdmf_L*oP zjVyU-c<1%@*9n5bzm~ovH&@+gtW|wOaSHa1Yu{}qB5dhNi;@V^)zm?&T1xmO=77GQ zu>^5I83#j;Qis7bhLb`5P^O#j5il&-=<~JR%;TZEkjqxVbA7FBSlU*IG}cfsEx;Q+ z>>JW%rPb7j7ZpG;b5Os2v`2!PJ z!!LKF5Tw_gQky6;^SYTe)BX9wd90)J&RSl^Yddb)t?au=65CD5dfm%H=#6&brh^}F z(nZXYj^t#vOM`q?1=YJ%1G$)~fPP>kPP)NnE+?{*H|3t9Wf?^*+be72W-dZ&liW}V zcn7gJ8tlX?9uKWEo)WXCR8HX|ota1vBLl^5wd-;lnPuwM?Z}n4{lY1tq+Kb(pGUss zJq8&5>;0^OY3~gbrh9BWE3W^76&l(1pMEku?!OLJt69f&w$*p9@>-uc^Qs{qtWG)o zX*u>=@)_j2_#d2g+h$%oxTW{Db=!mY*V~4Mw}y=^Bc`)^$o1_l&1OGlE<%C--nLL% z@{hKK0@Od+)4;Zc)O5uYclE4^54Lbk&88tS`g-tBQp`(J2@ij3UA(Ds;Y2Mvxn#3e zY=G(w=3POEwK^$7zUw1t)-gfG5u`Bc`aP8+^ukm>2i&t>bMf$0)oKOl$a=rKM zV!PwJJXX@sWhJwvBE7KIU}xD%ZG4BofC*976kr#U;r+wWDS{<^%d^Lap0&UtVT=7} zN>KP`Q$l<3znT&_^-ieA4gcAc(9=hy_wds1s_NKhFMYNy|1UBy8@COd`u&!LY2USZ zUX@J#2i^N->Nzlcc&fyp%hT?i`cpekM^MZ%AzYfkPy&-*|9>n@5Y`1o4?D+=^7Br2 z#^7ag>d`m!@U{)4sPhRILjK7L?o!o=tr&mE+aR2yCupU}adBaNBo`kUu|C!Z`E~D8 zd5S+~g+I`IQ9nq*lyB`71zQehf2rGBihrnEmX>YZIuMnkxmy_B`mgO_z`i4!{vmt% zGXWL5;r?$k%2N(oxUmt(Q>x!{GIJS^TdFWkL-IXbaq2QB?7V;LF#n3Z!J#VWHNZyoGR8(m+sCHrA`f9Fb##?>M* zrsM0G%5Z%Bqkd=g5}Q2BHETt%?-+4849f93tHPtkf@|eK14?0NshTFt9_Li}IPZax zBpHYT!u2LI!zJQmkS$$^(UCKBscr7cqzk)26$c43gi>5^7mWW94DXtls*y?IiJHz# z{u=&_*eboj)Vvf2dob({^Xc7#dYjZktI(|eQEgl&JDZC^glG`^#f zB;d?i6YDpx;$y4hfMi};N+PMcIEanfG0ViLuSn7XOqNl`7=j>lnRstTiExhDrBQ}i zSH>opCXkH@1b;?hq-HpSJLT@b-02t)l+n*tjx_Byw^pR&%YTQ09) z^!ntUQOO|bjjAY)qz)(A;Hd~;!lZtu+x6ozW(a*R`jpO8;Kwkw?KjEt;1MV=Ib?SH zo6aq(r2JOLGbLw&Wj{H)_!A;lNmrLVhe7_Cnnpg)q>P!x5#4Dfd3qQk(${$PadK(0 z1YrI!X(?J6d#|hW72#F%Xna8dO7|Ag`GxT9J`iyrVIcXIm5+d@%9NgaywW z^&a#$#i3cMnORUb-siDPV9Jb0(0hJ&Mh&$z@c%Dr|!Ti%!)@x^7jw2 zF@JK41#G+n?O|Y^SZNE3Y;Ur{1OmdX6YZN`TnVsKGk39HY_UnETXT-3buC;Zdc{O{ zaU~3XfkQVQ&B)rk9>sLBdj6R%RV}xs`<>0G9_3{>tXm{aSSB$=`MEp1ns65ZMY~!O zPt?EZy{}4-YB7{=FoRFb(_|@hlj-_!7ZF1w@3gjKI_u)tnADNzgGEg~}y3f?`^G7ZK=;$r^)ZyTYNkx-Xz7*cY_Tz+Slf&r_j@{J~bV6jhV-ueoWji z_CFN(+(TA-^ICC+fFgfI^9;WFL~!b!5+NhGBUQbq+;JFD^t6*#q@9x&;*&ZQ3of%e zI_tGc09h@p^=SBuup;H0-v?(TYR`9hicd)`)~)yS8i&0i*s;A;cm;>7aB+paiV?KS zh+VI$?&S)hG=o@T2b`hVPC^WECU?x0==vDfuyYi$=UE8kyZCY&kl6^Jixb)|a}tU(8ZvYg{k*d@-E7R3 z{w(F@L^&HU4JFre(){&4*{XFBX(o!H5&CTO2 zk3V-v&g~zj5ndDB*&rI_vsE`pNxf`xRT<+aIzRM;y9&aWCjy?DJUQM zcX?8wY>qKO))A5uuPo7<7m2t@15@nU~>Ypg4-cf)Gdm!S3 z4zXy1GG1bK%?zH8ic*FS;!9s#+!Fii`u3$aciaJ%NzD+g7uNb?3$(>}%~S8c!JVV6 zQFYbcSGE#*{zlfx z6xX}A#>cv?fYJtj2>rJ0rIN!ZnlI4k)O_UTVx)-NO@rYQaZs_BnsB&184R!5N$M^= z-c9hjTrBt;O1wy>VkA0F><-~I&05B^k=8KFH>&QAK_SyIDBZe?^`_FCzlE>FV6s66 z0~{7r9Qb?8cv14&`&I)^*9aQ^5S*izE_+YmCtJ>QAHn;$_#%fEokjpxUUju6DX>y^ z3{hI*<=3exj6l?<_I&3@-Y#JdV#%-W@%B97dN0Rf4@gR#RVU>p9C$V}Z-dR1fU%dv zE~ev!*EwK8@%R9JD_`265*-#=i>2&rbBZDr7V9p7!T8!|Dr`}sU zGsJ1=CJ1ThlDu~PmAR?Lu@zrnCk(ZaqI!Sirp$t7$Mdt)g_QK~yVQ~TIyR#QDWgSy z0#Y0h6>~#_;ji5Flm%X^>}*(x9*omzFZ5nAN1PAGQ!Y%Vfx_HQ-MMtc`rAP%w;Om?QO^> zID_1i5pd@lQXLr;9S0bKtB*cW9~dX?lMDLrjlxf+n2PAD5XO=HFUHz?QRW@6sCGxx^#*NJm7A|oR*BhPQ`wf5ey{m6tO zTRx@fWXM!L)A5~|M}Dwm$cva5F8yOKupI`ApCKsFs^bMx4z~V+5oth7N<8r>sMSVKmu3!XO-boI%k#y%HmFB0PZ05W_0!^bw~<>iOd!h zD#|L_n}O(392Wwagq?iCV@X0T{5rH@mI?6Rq{L?+WIx$iB;*2n*$U#+-3LsOuB2G? zS^7lK?Noz~i)1Ll+(jQn>$GMi^SD5&q8)=ab@3bGpL;|Uyk_L5zvX^P2BXlqqZhf2 z*cq>$19^^oH7Y&Gr@hvJc(;PR#G!z`#34bx#96kQ-2{Mxo<|W0rYj}oM1uYn$9EA! z=1ziE2dNzZp2z~0GDhe{Vd+Kr9CoaKfLN>xeqaGf1ELQAlBQ;{MoLRN#RH6!WEJy! zTPbK4i0QZ-A6HTaH!(MC2r(C;;{9ofw!<0%LDm9ygZ*O7wc=1(ICCk^DhR)vXRJA=xrn_A?>t6#I3``9!E~ z<(s$~z7|cNqF0fyX^KTMc8xZN422-G)izn$56;xIRD6cp5v77mKmJfyycyW&>3G2j zS25#2tGX2G4<67U$eGljR7SAlPe*ZU`t>E z1ChCazQgMKn}Fy`uh?Jzx&$&}gzQbj?oAVGkj~1k@i#2WL$AuoM+Q42%G zA~I^wu|aEuUOAt3!2rZv0h05Rpf%hV;s%v%<9?#z7k_S-H$ITJE21}Eb_DB54Ul)y z&%P!27a&z=Eg)6k$~*EUP?xi{Zd9A{4oL_N8UqO#jt<0BlIsi#Ocak3G1`VFTb5wkO#x)^{DfgZOuFCa9r#Gg z#OljH<_=H_AHVz~d3=b0=qwQse=oDB{)J5SM^rTQM?~2Jd;%^JQFZ541}Mya2~M~N z2rmI878r3?3JeGs$gsxfR?Z*spT~gv0*{-!z39VR9hd9Dqy=FGxE2)TqISU^FvZOp zB@l>9oCGfjmw5#rtQM5hzl~8pYyo&Be{{vby&1$cZi-&5A3+4LXQ@|+kX4<@lRRM^ z-dkVLA0LR%zRcx_tZkr>co8aVafH%A@O zZ|gBJZmV7~?ur-W`zN=J_gHrGUNIcykMQNNH{|J`H^|dez5|)2!-`v--4K#?;kh`1 z<$I_^~m}4StA(+374}u=m)cp?kkoY86tmQw7ygyX}!Cc$E-q3h|bM3{* z#b0B~XhcSHG(~Xfa-Mb!5z?%6?~Y;6tbovf_r@Cc^XOH3gS@PaKLgJgJ|I{}{WO|>$PBU5=UcEpyefD~n)eqIX#`4{=qnf@Ug~g=r55w}MMqv6-?uVGX zSw#_KpH#!;(_~oDq@9I12d6);>972Ld#o9G20;yt;FJF7M|z@uG4)~}_30b%v)hK2 z{y68kO|@}~z+<^jmFd<$LS*_D$1(i^JVP0;a+k9Ut~R+4Zy^JvZ@pa! zp_}W*xq!KNESk^|#Q!-LPwV%?@1(StfHGT|)J}iH6TDx<9pdTXQA<}vo&t_LP~liAfUqKNvHb(1%- ziOr&9x@7&zHfQwuJO(B{E;tPg6RC0ew!G$(4UDdUBo54_j`U}i#eOjYzcvEG&Fc=3 z*w@=if|QhjwbW-8?v zzVeJkold$Qew#TIajf3qcwuZAcN(E@iS+2UQzNkS&GP^mIykXF_Rry`+f}5suAFtQ z7O&D@$Nd438x8kNC+f#S&z)%ptYHG9@!O+c{X$ea9_U{i>(yZbdr`<%uSxo{SY*UZ zgX-kW^G7VD<5MY{%^jxP4Nw`4EUk9n3pU$A+f?77CK^i;EmU7t8cP>`msb%kbE^#< zrJJbhSDL>zc`D``6kf8r;$mXCnf@jz}*Jm0X5 zzB3#PxEo}^Z8eFd#TE*n5gp0*A@ACRy&a)JHz&+&C8ghHZD)Mx_cZ`>^M$xhJIZ>2 zE?la+exzDy1fL^jbU-u#_EnZ{nQ2 zBme!_!Xciv<1iX=H+Kw;I~9_ico@{4)H2)$6ikjyzqL>s75i@}f4q~sf-2I2Yj;AF z$$(x;VfF$)(GQbb4>wbyTYiT-y-pC&9wgcL8;e3A0+Z0P=UM*^H$8Fq`ix_Q)XDEH zd{zSWf-s72~Y!q{~RnF}7-4Z-TY9)ktp zFq?r9_5^(WKm6HbeBGhC{kMt7W*bjlKXATLAp|trjlZbQCmM9!0UMA!gLDA9466t%@_RiALO`ZgN7@DS= zLTjehsba#{&tEZf;}2F~-y+u}V+pV!MxM_MgF>XfWJ+$c!Q!BH1P-XfiHxU+FNpDm z-bCmUVo+2lApE1769H40gS9cD=rRWbMxXG`wxUs@MUoZ4#zTavFv)g3*nVeOxm-FL zDudH~JI*n3vHqJJd>}gKQ0kWr_lkx0zgpHqhV5sqM;eRc3o|2eE`H(sp2!zj6zRwO zYu+dha>)07JB22)4^u&m9z{J)9Jfc_|yPiV>X_6ra zHM0R`y)N<+D&-D;1f~}SvrN}NrW5G*<=ZQa;|s4-Mj}F`ySxt+0mW>y2nJsmN+3>5 zQIR(FC&TIs?QAp!yapp#D5l31pJfrh;R;WwHm)NBVyJd^D#93+bKDAVz*%`DF#lDl zf!3>14i!+_xM;RAZ)Eov#w$7}ae^=ytknk@`7J)s;TtTBP?g~{@IoWDOhnM#WJ5dS z#A|=@Xuudl>hY0rg*!%l=2AEzNhDcg9QBYUx~Z$^Q*&*hI)#pcRjstmRs8X62iaoT z4eOXm!`XwQi|>LYLz#_+c+%pZzG0@+NlWK)^}=Gbo*^BroH}b=1oM%wR=`+*_g6?s z_h$1e4LvDRtnx9jkMg+aw5F}U`W$Y3)|qlw*g4}SUy6L0+iJgOxf{?+hH4FmU_nQ0 zI|o2JoMY}fH5_+VctM(dT){3H?%k_wbCatV!xXxD;K=@@`6c)abAmj*I#>)1Hq|8L z02Tr!c*e_(IUCCe@Yk|ioXPhQl7&A)@3|#yK43W_b&HH9Gpdxz-b2g3#2$a6{1RyZ zKOLkB6+9EUodZRIpk@f&QQeqd z)aEYdNR6+?Xn+cZ`O{Vl7C95OXb4*>7YhWk?$22;D>oXPR5D2!-#G4lrErs&G3wK@ zj|5TTF$O}CzFK7&gqhf|sJ988jYY0#M}Sh{!%?kRSBB--fjAv~%9b{`KGYX~)R?#ix+sbltVF!g_MW@nM5P6e3G=28V z!i*+UgG>X8fiuCFlE`U2zt6|zr4F~M3aXoy9-Wsj5%o4AT5CXQmLo)B@KKM+P^rdx z|F^+3C~a58o~3Ys+(d;I2V|VnK)B@o@}^WWk4jZWmG1D03sqZ5R+W_PipIQj9F^vk zzjo7W+OK)b7J()ujfQHaNAUwhNjex{N#R>fv>NQ=LyD#=8{KEh{IUDlt|+q+X*_qt zM6avHH;Dl}xlc-gv-v-IwE~Y+n7ma1Du%J80m*QK_w&6TpitGA@E|;hk|HW^JctsXCV9JWT^U1;8zOGYTu)+nJtZ$1xA1wb`+`nFy_KkB9(rR&F~M$%RWPT)KYClF>c)`SN@H z7G050EWn!>nA&79XQONn%gvZn=nf4x9e#vBgY7g&`EpdUepe!_=xQufMB%Ns$JRUumd@F01?O00xi_rA=kNY;#v~HwZkVl)u;PL zt86`zd~kk~}`=^O^DExx0O6aJ%{gn_Ds!SDdHEGtei=m+eeW6vQV=AqXQ_8VKGsWmz$2!eQ zHe!vCSD3&_BKaUoICf&iXmF=l&Z<%Hb4&O$enZ+KdyZ`Riv7Eesq%KEKN=?A3;v_p zvbf5!i3oMa{&wu*!tu= z_EaU1;Z>zYu}A$ja?SPB#ho^S5}*R7Q%?G99vC&#tI%(I-Hj4*F1Zyjo8t&op-m4~ z$rG;BYSG=*hKH)ALCRQ8)+KN7;3d6|8(?n` z56ar$(#La@d7hM&jymk}z^khbZ`qUW9I$dBDtf4og@m4EBd$UZn)%5Lo}XVqLnrBt zLnmdGLnqjUR#CL^zQ$4A;J<|K?=0hTc&BlT*gCVN_a7J;qUY~7SY(&kecm%0C}j2A zJG73~#-rau7+j}0&%CmvUG;7`6<`#G^5dLL{r#rcrUHuzsic4JW) zJfVQVm``+*S21`|Rxor7U0^pz73Hq62?e>rujS5&3GKU<L73H>b zWbQqMv~(O++@>&$EnSz_(OJL~!*`*V)>iGY?ZX{S$TH}hM8zZe2~bO~4e?Taja(;Y zw8m|s-5=ODn%hJP`qSh44GqNk#uM}15DXhP>hHoMhG|<4N^XdhsJ8@8?4C4W23%u3 zBI6TV`1@;XuAFk*l2s=7kDyPHuL7}E6NK#uq0=Nz3W?GL7|jqYU6xzOwr&+&5ya)(+g zFgNfT*g-;BL8yzw8Cj!+8wQ0*|JnRGvd<2VeHkwXIb%f(0*UAN*VvGU8|WYeN}4SD zMGmEh`jwF^2BYo7tLAFy@7w7hQgQhxnbJh#P3F9$!j^BnD`$9M*!IcJV-s?Vy`mBD zJ9*?!`et6dy-UE4$F1&p@MMWr`%o|8Oc%kQKzFz2R{hf7Hk>nEgM=o_LkCbrEEGc4 z;D>Y_MwP0s^DJ)Z20Qjief$7qSezOTqSOmDNL4#j-SHg{sAQ-_UMDk%G2j{g zo@SW2l2kc13>-imkcgk!m!*T5U8oG2xLC2Od9>k=>f3atFB>G=+TTQY;9o^3x$)Q< z_KcBzW7zgkDmzzPs>w}SKeF0XWvEAqY41vV=%8m3q5?HZ2OtyCt)eNc%Z#;OwJL)< z&UStqu9~EgJ|q~M%COz2N4BF1^xT0HeXxVycD_k9xq!>|gpS_9|hs-qm4V0JF z{>4S=DY()FoiCl7Ybp3=zW7lV5xO@Fr6Zh{Y01IR zcQZvE(5jugiY^ z*sVu~ZsFOtwuhO97|uq0M$3V_%wN28?pPWfy&X+=a)b#RBx!lI72m6+U-ff0F9y5F z@y(g-CvX@FZ}OI;FNI5tPq&rJpOHpi%wl|^>9BE|%u2Cq#-0{Aw|BiIL;-a(!;}GF;a=FX}LJk>$)z{GU1>Z~3U@osTFG^I!!0b^@w9 z{s|!TV~wVQZu*$9I;~>q@x_A3g(vcZiTlzxHg1ih zFB{rgi=PN^J`TH)xG0qV91Qmf!0?v?lRqMYV4w6oFv1BWd_uxB{NdP(Dl118wdk9V~}QbrQkn4?GZY zZF~OJzO1Cn?#Ui}#iKkbH1`KzPCeYfAf(rpa2-WHe>mX%ni?a}*=EfY!m`rsxH5v2rpn-3aq_ZF3gMf@P14Oor@rf3TjtME55m@ z!~JWLQIRo!gTAod7fx#*T*gPdfpkEq7|00i64f9?p%{ksax4eQf+<8A2|`)b3fQy3I1C`G zU7mu35OC{2khkJ|{Jy=JWPIFUdcD`khvu3O)-J?f&_M(=I*h0(kLGKc`ivIE_*-6AUbr-32Y+}NEyW4-b8S#Ld7lPk$bFX|{l zAV0mx6@dg#F%nk&cww;MnXyb3_&Y$mIIL8e(u+X4C!a{rEC%YXj~ftUgC8JU;|6vy zXHydO{?bh!`UThhj6~ICTDfx;f0j2eF9a-1fZ=4Z8I3HUy;qj(p6WZ1Z3?@PZeAcI z$zGT<(80&C!<3qNkR`0(B54~^*3DN+&8T+^lFgW2+96Q-7o3yj`I!2nRm;{V6*^NB zbc{ld94q(Y}jierQSbyz2K$YqfqCg-KS6 zYRC#YO!k=_>+cULm$Yh_4=vW4-(g=Q&3>;VJ@uo8xAJ-)f~*5BGutHjjx%}|PG-kf zV`oq93Vsy(J{TcW0b@<<@O|HucoiLF+aQv?0<>Oe^3OLJQJB$z-C?001fXzYfW7Y* zbGv9N5BK;WXSxGOq+f+{*HDHSaL~?km91g~<9CmKHPKIQgKT@}b>1J_Rz?Xpa1*{f z+?}*el*pG~%nniaeW18gnBvPms~+8Kb)m;phivOkF-x=N4Ifwsqk9)OTW}iFyo_T7Vlz9qt8t(W|t;{^HG(eZ^ zF)KiK_w7gYBlD6}E(WYE^AZ(rg%MAfXS|++E&uMp(C}GBbjjDZpuNmyh7UI&8=-Xg z0O$q4*aFO8W2P9$1jx902|`*n<$vT|>ze$37k5G-FLe@xz8ms{UrR+uep+sXj%Iwx zf(Lz6mA_K+D~xwM#i6HhIH#&=Ov=3Ee;%DkGep z8~B~jYdUNOr|o`+=21CPBub>bG-d+h5bP*rHc9jCuCu5X@lQV`)7EsG={#zOs|!H?M|tSzA+ zRBHr}Z7WTKYSAu1}ib#mo8f zNn>aq+=#rv30^6DH?0BOguH=`TS?-W6N``n-a3Ew3)PGJv7+cENAxoga7q&8pe6qF zm3<3+Qor{PX8UxgTupekn$TppuY09_QllkvnT<{|Roi`Te3=ivkk5kSGTX=aFV#8L zW$w!?_|VeQ7^C%HziiFit?}^@L$kLuo)0b_1{@{?`4JPgTavfOZupX-eI51I{bcQY zynL)f{JKLbSswE;-=-t*gHy9^;!(dRK2LQY-w_htQjWmsPI|@=_U2Fg(7C#ARL_P> z4r+(w_|UWNuPYmWIDKXReBQIR9&a+LTbhJu*BE`a(`DPe6wj2Y z$`D!AW}$Ke$x&r#TJ_#-*Ju;EX;fM-oBUP@0Y1`X}n zn+DmVol5WWK4e=`E;&owm%I&^&Fie$uiUn~j9e~9;!uXYP5CmKohd=(*$E9SbFdvH zV$5zV9l5NV;TNDEFhjd`Ph62!$t6*iRykj~O}Y{=<=_O8C7UAYqH#%kQBD2zDBI$9 z`%iaAiR4UoM}wxH9;O3Uc1NSdR(2=pl1rNliBjd8D~XZITN_Nt%I$y6D65?lr=#32 zqj}pJ`zCqYtRLa|R+WkuS=hcNMpSTmsm79V2gY;CX{OJDJlfLECiy)A*9xtbBtkL+ zOZr^cASNZtnH&2Q`U0&FWdz%@=As1w0*mKS3?#CkX;a;4UtU0V_b6Yx-4>C{t9+7q zCX-1Hg6HnZ&XYkQr*;%v#G{}|TRG{tq;WA@N{u-2B7S%X%*B`F{6?fqh=(a*@uZX> zqag3%Anb0kd7j$RmS+qc-Igz7Om!J!r$zk(SyiNa+Mxrfk%toTEcCJdI z&H3_DqRse|MzZZspN(Xj^SjU-BKU=9JZrpYpU- zj_|X}tyi1{+VxkO1s7%J4tbeOsmiufjz7D}zEtkl52K!i~VGJs~m z#3?JG<6ve}Mg7e&_$U3*9OvOi%HhKIJ~1-DC{X7~!pQ_m=H#N+SfKBUNJjeBH(}m8 z?0FIvw9E?){M>j$ol66R;)9bsL{NQOXHMWzhBlGC7tW;R`fYL)5iGOd6n>? z=1i?I^(rZ2L=%CGR1~#v4JsvVZ3u_3MZlGMpxgf*Zqx_VJ{)c^Ug{Qg<48g|N*3&$ z%;@t+w~W0|T4b7WtE@I1tfON|zwchS>ep5}b5dibY19{g(X#;YhXGSLa&&(D@OmWB zN2gICur31O98eqlvq0GW8)Y4LN&~bSeV>C(PcAp7~vRh*uM^HX~>r@Gf^6n>BXzhXoFbC8$UUDJGRso90q|o#$T%3qf*IJ@ zKnf?Z{9lKI`e->Q@fW2kW&&VH?z>h&ipH3?55HOdO0Q zr{I%0XMw2tJTl9i+-8={Y4|@nTjTy;I$OPg&LY}fFy{C#dx#Bs50)9dnd|RUL9;0D z7g$4sIo2nQTQT3kOzn?Hw zvkv7rha$9Q=MVcEwdjtQsL#MKdAYMOx~r#bC$t7xo?pnhJZo<>)d--GY>?IDK2Bj= z3X3`1rQ>X>G9kD}EPNC8U3r-k9G&XE)X+TG5Y{NkB%Z=0Ul2Ah!Ih>9Y4M%(vg-Hy z&~MJUKg5{{K^oJ1+(6u5HVTckJcjmA(AHrqA_&Gd7@_;XArtWHz6TCjdvQ-Ag`g)* z$N`{;ZTgILcz8f}f?#BcvLC-9^isUBP=_P6AG*}sR4hIp48s){jT0$MHD4wziplNx zG`Mhu|MuHC-@L7bYq6Kr`8OmD!eeUTBi=p%est32o`Fb|h+6d?z;f;L1$upXY%(bM zX2m_(+Ka2VGI9i)%fMNp@>>P>Z5qq1ZX1UK4J#Bj`AS#mc1ANDm!bCg*2Z7 zCKXf&=Oq)7@4j&@MusDnJ*VZ4VZY>JM%n*pt&9j!uCGqdBsCkro}{c>YwCNCBi6If zq!_bX!f7)Phn8LDZXmWu9@iujQ<^JwZe5TLlryQh3*<`}Tyy8D>8IY@*+ISM-9Rn9 zQoJ1SPoDf*^S>+tc2d}x_3GiAcwg{EAIP2X=Ak_0N;mRxn>>m@zGXkM<$6D)urAU) zka6;aV6880Za%m1lxQg?Gi^@=+#6$%ycAg$!2?pRJ#3XFX~yN=*TX zid1cbLVk&3|AeiosudaRjq0_vbea~PYVv2t``J2lyvL8iIl&Y3dq6X9N9qN|Ikdqu zSumrDZ#i_nZ0ZshE;D_(j?4anPLBiSXL~bXp~rc>B$HjqdsGN=S}Pd!g$O)q7*fzo z$5NlOA=EnBw`r0omj1D5Yv-g%){_fgMTR#fCeP_q{{N!Hna8C6Ww49Q^&GFGy#LWN zLCHu~*KI`gm^~o95*I6f?a-KIuPH?lg=|0vjm8i!H zJlBHn1K2+jH;&eaBFvVJzjyPKuaqxO&Ft)A|C`~y`7N5C?pn{WiDJEK(_w{jR-h^w z(WpL>rHWSY#Ph&&%0@k2FWatjt2bV6`fcTQVRY_1eU;-XXWe0y?fhIhB{6(<34G3g zO#s7T?6owuol&z9!+q;c-?RCRj66UfMjLL1eiPjo)+6DI1A6C5yqBo0&ss-Qip4Bgqn>c1>^5Ac7p+*z6b!*XY(J4kxX3wqFU zhazzJ3v2@W#J#?i>&I6#A)pqhnY;ETWRFDj{i9Cm3^I;T|KricnqW^|VhnClg=Ff(7Gh7_cm?1F-Go=TGSkSbK?4q*#Ej zw=IS=$Pq;M`Gc^q z9oQG10vwsh92$lJJ_DqpVyTkFdOR5~RB5uRhyHgFHWZJ7qOyJChU_@2QsX3{>OJ#4)U3+g%z0&v*-pGW|IZ`buIh1k-6U!x1ws5PBH>JuA2rUVkN;Z?zZ| z-L@Y{Zi!@7tilx3nNd>xab2te*}Rjm{o+L0uZ`w=zu$dwjs=P>*1NNAJYAUd?-P54 zR`i*pmh=^WD&# zywiZwd+pNzH$j~TGMBQv5-{V~5-?`c8yu}>3Q~ga=Fmc>h-Q=+Rhmu~9B}LyOLoe{ z1$$>zW3inRYSGR|Ap{RT%-0(>2Yv@EJF95ezxTr-gW3CTH*90QUhIN9Vd@rwU0&Q& zfAR}1JdFR0Y-C3n)ej;mw2-?8bPSGy<*`Mv2-1ykQwr8-x>*GyFbgh2>5#R_Ky^uD z{;3iOrid9g|DuAHGAO4OW|Y>2-Bi^jmKjeSoU&H!Wuu%U>a*#@M*STk8n>^t@aBSA ztL#K-o$P$j)1ZrkTRmwIQoU|i@ za$oSgi=p4$k+u$@r#d)W4-5c}u$DgnpBg<4SSmOu+WpLJTL-j?D1izHBb_Ptp(EJ! z%6-qYXeGf|j`2U8tkdK`yX0v#znDKqV}R;Rvp6r#G?Bo<>qwEZIA^1K=*~PM3<}}3 z#l{#c-x|Q1vZSHwySCF+*Ih*A4b|6+8EHhDH+^|>X*TJlB6V{JkoEn&XAS$88KAB zy_*ai-NY<(XyWg@?hw2(3j{<&w{*JM0r3!X2a6BFW(Ci}9%cDN;(D<};tU**7@BvK zQDg?t%541A)W&dRPNRc;VR5UnZC^=v5%vj>;mr)nnz-XULpqxiR*I>&^i)W zb^wX(hGfRyLpvs6LEN5vr{UYy_UAQb(Sv@ad?g!s8-{NyvRO zQ8}Grj!oDVmV_%Unq?zQ7i~%g7a;ja|0Xb7HT|`FD$GMM1+4D;k@bjAHZw->P;D|@ z2oyB`z*fjS4#dS4=#??fe%b8{;cz%^Q22|4#lngZn^s6vh3lqjz#ZWQRFoUhH(nugwWU0 zqQo0lJHEd$BttS$i|+dWazS>_r1kcP=Jj!h-6G^Ip{4(O7-+&|dw-Zhsw)c!t7(?$ zY|T$QA_fg$4WG;YhsROPq+nj6VCt3Wxqf*ocYrCq>huGCy#<4sk6EruOE zd5m*iLRp!8#BY#7xYhbqt3%ZQwEsg?-F*8C9X!}^CyEe@&|iY;CNHQPgb$fNJrZ+= z`8__?0V;GwNIzU-1XySWT0U;6j=Km6`>Mjlb3#@$q0juAWk!W&(k;?=uC3!y3 zpzOVPN0=d(Jz}6qFim~~ZQiC47>;AjzWi@A++cC7JO1_H=h;Y$PBZ8*+@5?* z7hyXD!e2CosSAh%LcK5NfaRi%RVgOomW|}!srOF+xN@OQt@?XTta4T1U$qOHg{Q0PX#@{AnD0N1txNuV1T>B9ibZYj0#zuF!l zD7|sza{u@q>YqVsVH9q0@JgWGz*!;#z`us~d27V5rjz`*cwbC-OR;% z)WHsJPp%FKd;+Wr<9(i6O6SRS;0F0p74iU^epFfR=$<(0v9xA@bYax>2 zmKcKSyv zu-2XMLtGDqSGj$SP_Alsw=|sq`THjoA`}v6>NVf#3Q|KLHq97b^bZzI?-&5W(ENii z_xS%Di)3In{D%n)uPq0vk-a2-VArKeS`P41)hC8;it(B}-@muh!2;CirfBPDwmeo> zgBInVy7r4%bseL}|awPYm!;&O+-uV$mUsp915fr_XjlMvj-wtd`Aec zA&K$7$fkz6_@B4`p%s20F{lqDwtzW6&K}LYu5~uEasSuhpMLjaz@WSSUM{OGLq z@f@a6sf)MD=sP0I-XiEP!1WXuTei zQK!MW&QoZ3*ZlSd^M;+G@mM^mM+FOq6(yqa%tUhFJv)VN&Bdxi!h!40B`+{#Td3q}B()r66y>DPr(IqMC$U$fpO7R*_Y*i%;Qbcx;p_FL8d zP+URBMve1s3MV)#c`hoB{oc~V2ngQT$h4lXP;mY389ZQ+ATNk@VgFQM zdJydhpAtY{Oyv0_-WSiU3K?$r(25yTskOqt`7>$O^7YB8M|YbPy+LH;MjW@1J@nb>(~B&>(v2iap|#~}Wzk0K?;l>5xj z|HCQe|393vV+sEG_uo0i@ZUM*LbHBoI;_$GoC6HMJUq&}p23m=F$kirTy=twbR2vC z8LH09z~fBwmu~K$)E~!l1A1n`x=H%6`jLkQ`FwBziH?*9-iJ=Txh+aI?n!5138suw zA*DA;rMxgm|FAq7OU!05I0V9gOQepGfXd50d+xlrHiPw8?ii(!89WbXpe|M-`yE0I zX|kn$P=4L;UgNolZ<5>b1UjQV_xQ-}XG?R$5Rc)11zqN^0YMif+kb*CQ<6a1vltx1 zx+Z1Wghr^H=9)Me#U>*gUI_eYG^F=SHtq>#1Fr z65DHc%fl(!nnNs@xTjj{+1bzEJliR^QHnT`75`OJaz3olSzT8;T{p7oUMl{52Z_$y zaC=i*Ywi#ah2@-FqrqEpryq(%owZ&}3u3243@Uwd+OKtkaLeEl0KJDFS!V0>VHlB` z_!$Nlk-T#*A;M2YUMWP!Y^sV+kX-c;FUpqMZVD-gWYIvf! zw#_k>HHhQ*>?`qKO#O6XYIrJKnqPs@I!*-X;&uxt$gH|ABW-%oYop{ zKO^IaUQ5`ClxSO?SA$lMHkyF5=Nvs3p0Xc~JaV;&3d5kk0>K^*KM8yXy2bpuOZWof z8%r)WUtm@NcPobpW^tPE+EPh8FAJ6^LG@|BLCn&X%j}Y6-Hhs3Of7rb6H;n zLEZUGK$-gU%K<~;qvpslv2_HEiWB^gQ}2NOgRMhE4bOM;_~+)+9}iC#6?F$0nR_$$m!=L?cyySlrpJ_27DiiPsQ;J!X$7XUM*(9h@2~JizT9@c?ZbDu9VJYy zA<9$u_8XOzpFoYUxvsnGlwwSz5_MUXvsOTj_gnML7HteEUj^5t7XDT(XZ-7RLrvJr zV+}K{&B}`VYZ~!r%S$YyiJ)2mz&tQI9jiV}lIEutayF~H;ZzViG=_`eq|QzvzbMUE{^XRs!(s z$(7zMxA@fP68+z{)pL7l7R@FFl7CwuP)_@R;wZDnxEOP#C5R5=v?Tk_!M}kZM(cKe z7hN}cO2F-?dNez;fd0Ya_xO$LZPkwN|fRl%d)xP?S5Y0?x^*gAO6!=eqRM{D91f$5-{hAr$3SjeE&IEBlx zRJ|Z%N2xPyL(#B8v zYUK~pR=4yPS1v8{H4oh%54+X-k9;@%3Gc%K2-H!Koje=q=-n#Aw3xvS)b?{3)kppLbUCyK&-)J*G@CJXRSq`0vlVSu5ndpm ztr&zQQF0v9s6NB)id&d4EUKoo+q;)mqIvXuUB^W}c4@nJy%ApzMK+F!=S!A48Y_&n zlxY6`A*d(;6bN~nAlu)L`pgO1I8=1cwl+tpjI^G7db+9U(ymT?eigD0${c6spWb=T z_m95a!j73w_v66K<|HdM1Nm5y!b8KHTAsjMe-SZ?B{=A#|`UbR3L>}5;hLlZ^+6;agd7Bgf0b5 zo(QaUuAf3Pl)`9id?m|LT9}Zlg*v}!RkRE^uK1m|gd_sZlu&~_D688+r-q->>~cTD zTSNCiyE=X=ip{Mjr_i(vJL0(G#)*nV6k>*uutQ3L00i23-Hs++)%r zQuVaTsuD`UaWPqxy>y`6MJAbWzPalXpV+!tX*dgCF0BK6XiM)sl3hn)s$fF&*Y{`8 zdv5-mC8r!)^ovnvA@c4BxH4ds0%aaDJvPBmogtNpoKUe0HUR@5?bfTK=)(0&>olwO zaEjrvv$sNX4WDOSYbOumS(wj%Nc{pDf5F@hua7Oc-h7P|O< zNPEkmIM;0p7j0aF2X}WTxI4i;1b4UKF2UX1-Q7J9+}+)s;J1^t&e`k8t-DX%9}QLg zLD5Flm~%X1yg3RYnJ2-&q6*qAqq-S)A3&dg+qoF+wIol{Z!BhBjcPKlQtoPYM9rvY zi1}$cA8qAyR;&vYt9o2Z^4nx%4?lZ;AGlwGLXr49R>f)7IBh#RATDTx`b@Vy(Egbl zl~;$6iv9osyON$^nluLC{nj&2nV1YW0t<(#F^UxmDd@#y#5=P6JDt>$($v~CQUU0v zKoV!&^RaT7T7YNX$NFuv>1?IErXX_9c2yK`Puq=b+1bg-?_rg5MV*d#-;aAm~` z((__0b`3g!S>V9h53a?BA3;6Afscke9L6QvM4qT)%)Sx2ff|88wUZ{EWHDDT+u~#p-a+}m z>0izSUsD6CP!mv@da#uH@#@Rkak@cm2*aR~Wu}=`t=q>gVS^Kd&+W(TyzNZJ| zW)i*(WKuOVldlYUvZx3Yc)L3~5*@bU=Iv0n==@dJMas1KI_p#Ab$wH|un;7^XL8rw zv}vg}adOoBh@K0KM`$6u(C8o>-ng;&J+)_^emkYncDZR^pV!?iO;dsz^A^dbYQ`jR z=TOcgMYjbI~-#jzOweo8+-YS|1(!?BbjOM*-zD53$2;4pz;TaV|(v6+Xq zXxAWlogG_3v-v&#ukFoFR(bk=`!bCy_8+J}wV6v8|Mhde1bsSHRL_>_l;tSBu_oVJ zI25eyx^f0iajd$E)%{2p5ncI6%r(3FQV+aT8t2uqUObMDhK4d4s=z)l91;)Ow7 zPe^ID5W60n`J4COyEhXP$8j=LA}^{uU_z;M-Ee0^b}uF7FTlz^^#@W1+Ho3%!p-DW zb8Hf5ZCT|SsPJe51Q!O(`9>iW?MMbGz*P&np-Zy%$2cl@>YY~DRCW88GTjelm~}M~ zA4<{RSju!tq)g_kKj#hnMg+vAr+fcK1foAM)qU7@oK8oRWlrQ&$LWI@!wOh$IDKqr z>M?uz8%Q5+c`tvq|3&>J0Eah)jM-B9=}G-L*4(nc- z?FaeL?my1Yhel>|vnOY->{%OlxbUbyy3I{ z?Qsvri7s{qcNX0u{H!gHvsQaqs4ZW5U(Q&UEUNqG&~mVN?)2c(aJ77Kb|(7IKso-` zKcG1(3H7d@t2O9G2>u)S<52PqVLPKN;9{9%bZCkY7R7;bE9A0K*wdj^$jS)HWHimx zNAuq|x2qD$HqdeO{Q{XV{$14E4x8_fy`3EH^Y5bO-&g-5P);7I;f@~WiT23__NIHl zA{s~KU!?i{MjohC;4=TVHecNjAZp$Q@h{Ze4_zDr@zXAS`+vgb=37WTf*~(j{q!XS z0zp1k3kx-Zrdo7Aqi0P&zcBz&^ZRcn+Vna+1MRZX?u=hD)W6h~82#tmf`3SshqO%` zTz&Qx)pPusKRgW+z)AN8+1&78!)V7KU{7@RH;tb8XKpDP*#8NskRtcLy59MRkXVWX z7Qbz81@LxmnIArrAYVXCQ2qyx4uHw|#Kq}Z^L`1!`HKFuE>t3XN(@zd?-*o?`A;rA zBmh(2Sswl7pIo{j@g)(^{7%NDXSM{#?Fa0GJc8P!zFZ=EfOd(;=b25_?S=oMfcL9+ z91hT$_cy!n9nMei;yjN@wU$rimSbOmX<`4)^#aVFuLASut}_MdfAZ&CFUq80Po6m{D#d#7XoV-h@hmn$Ic5(MT(%5Ah;I~mp^Qv@>m|2=4 zF{R8^D)IhOzG^|ZfEc86MuM3TfAFlw@2=#KX!z8ScyF}^(jJPqf8dM*l6q&H?Eed16z4U#e3j9>XEt}r)e%Va`Ss9_FxK_!U1$S;Y#vp@X zMid|KE0kMzj?7uvw|DIh8L(q-1JAck)hsqB73xB+VxZEqRoLR3k2%wm${YY?U(05= zgkAxTT~A)KN&O(!2`18NEIz5A@g_Og-HOAQKR{GA*anfkR5?JK`h3j$nff zZuc;ondJ4eZ%#q3;7G0e2ZZ2|z>|@4RG^7SpR$tY-byRXM2Zd>qp6otlq;|>*~T-2 z?w=p4B@B?z9)pyUfB+!MVS{aI)eWwD0B(#>L$>wwtt zK%BFMp`epL-*=?-STv05)m>7njx|a7iPAuppP%sQ5>%CKE>a$4nTj=y!L|tVb^BFF z=3zWmXeAu`ip>!3Vh@unS$&{*PoNizGDs%R@IcEvy-KknQCY+8+7+>RhKj_y&(jHf zE738pPunGbX@S6UglA~1sI@ikxv%YdtGYk7W^fdz#pVlL@&xvfe99?`ocdYGUq-%r zSG}kbpfmDCrKQ!{$GnyfuD&uE^l}e zHSw7PM+=12dzUZ0zuv8P-k)jTHtw{}E;@g0%VWK}Qg%UpRqIO(*=o*6t96iy)`YKs zN}KYp|JLLn?kb1QcveK$)GLTk0lzmAE$3~q7ORW+Z1c)@_{UJ+=fA~249{K%PvDJe zhk7#RAi1D~J!zB3Y&ZIOR+BEi_PC}%0o#%+NxfKo`gv*OTjp~%Nf7cx2@G_7W*(P#ISLROP1*o)>Iy{@Lr<4R=D`EP2S4$6D%KcC3;>;eU> zi<$$Pq;E?6xAb}9nrcXO17q4|PWVr*42}Is6&sXJ9A+xb8pD~gYE!{kNfkPWWX= z@vT%`59|6F#PjZHS;CtBCtqxP3hM(Wx>DElx{5Y*rAQn2k?{f`U1_0Ft7O)KQJ4D2 zI*!y$IZMCB^FdG~Qt5_9F13Pn(CX^5LQ20^m@T%=h#$!WX)=8>T9Shwyjl{bS!{^4 z_fzL*>KN2WG583y74cMmDB(AjJ$`ZB0gQsvl0)mG;Ax-0UKAdN2V+%o4gbz51kEc} zv&oVvE#Wvm-Lep0zGjuopEu|!Uf`HY{f8gImavJNQs24cS)^>zmkSs=w_lu z4Gv6qrO>=;KKfcRuvr-mC`<08-VJN}pUyUWoz~bvbb5tvP3d9zJ@#a8V`oxmZ|Efx zjJPpUN|#$ODzN)Oyid7imVE%g1fcnT-yjt!oxX*}xn(4$e`+ro#U`F)z|bK>r`(R< z+8b|VNK!Xt*k@f!Rd-HV@}TgNW@XP0x53w`R7p}in2%t(&t3g_OXbQ+?kK5mP*OM1 z?lED#Fv0g_>ryPs!wZEXO9zb`9twN$4g0qt1tN^t+w=nx3Vc+Ur5sE^>zmtCxsX6s zCIEPwFDk+$YdX5URjhd2o_LGx_W?H8bUqDFROx>Ch@X%(2fpD~r^8!S)EA@{Qh3sB z$#z-g$d)~EG6sEo;|7GNE@;=rZrrro4N9!#@K|0@dqJw_i94aQ1ciE%8zBG}mG z<0Z_?xoB$J)LH7ba4gNMxe~N7EWOezp;_0{QD$A0?oENSMER9*#H|vCiBfwDkjHjT zQWEaEN7(n2bim4C5#xJ-tYIw4U!(gS9VTwEE&K!k0?l>cBaLMIr>a=+)43GQ1h~Hq1qzI(ZPgTJ+jidn=hj&f@S>>SF!j$0|3Ka^0~( zF}Xw4UlZkS1>6}aePR_DMWqwmc=VDVlAuM2o|4f7$NNXeLuND_6zhl8x4)Xff8aDg zc3Y@Dz}x^79odMPC4xFzlR~}Mo&UTl^FTMhH7xw`w*#WJk7soYs( zNq3A7mw2{Al@FQvQ0_*bY0!9EblUS=_q&;RwY`%NGffK6htSdg5?k-vAnp2}bVsau z=b6GrdgZR>66lBjn-$_|x%c#!6;c=QZ(1SR+Biu6$CfHTDfXesmZkT3hzd#Or!NEg z1ds&jy+psAkVdHg)d_)L5R$`AL!7qRBJ$~ngo>xb;t|>G7C!HNCNe^774j9oC3j7!|evsGr*c!oZ5R1*S4qC2m%%n-{WU%^35M!r|_hyt-Xd_ zXaX};GNN?mAw|F5kY5RWMovdnhV6yHt0JZ>c7^Q8eFz<)-&EzwQa&dz-H20Gf6i16 ziN&F47=6h9Jzm_21P~&U5UhUXD@5V{p*~9@P#lKX16tHG=IaT8xSbFa?8>jPbqnM@ z7hUu~@J26YX!>y(V7c_Gsv^{mvK9tWBurPgsc8%xR?G^4O#}kCivTOel#u6sWRpdh zXVBdp;A$!5jpvQ(%gRvJ@H@&oa~HhBr;t~5>c@iyCazDP!Yx)cYnE;4g>I~8pQj8s z7t3mEz=cg=5=z28HqEluzUqGM3Rzd_w5qNFo|DN^dk~! zU$g(Y$~RlX^OUR|zdAyx<~b10wiq}XA4i`V$U1V0d)GL-`EJo)O8#@Nxa3j6O{I5O z=c-}G7m43wO<&82B`r#lT$`fQdgrEL=$+iLVOcx9$PH=E&B9%I^gs>%%iu(rP0}X696>?=PEW0M1e$TOzuBt$yB@PF}Cb`hz z^W_ct?xp8of=X(LsQE-r0AbNShZK!_OnUswqDheNG3ZDF<*? zIQlbPQlK2eP4*0)p%?F$P^u@!cg&Xf$}+}+KW8|Y)8izS{_5Cu$Z*}V^U{9KaU|D% zJ1t(^YY)F-C{db12%J*ai&We`Oob+A--*hET1aK7NSIsZRh3~Z*=PG^-En92iH4VZ zr%9PH(n+ze?8aj-#R+1(LdyNR_;y9=MLlHrhj0zX0{%BDxh<19rRL}_3utmX7)-i0 zdiUe<*OumBm;@3G6yq*ed#ioDD`(sZs{3Rt9k;9wyI}9JVjc5MStBRRaiPvn8Y%LJ z)?8WH8D8}>@>f%G{L4oaN$ViUIz54)LyBMdVPo3Bqm4OMF$0FAr>j?vu4_x<)D@fX zrjq6|`kZaTPx~1`d&f*d?XGKIJz@bfk$lX8XNM&2_)Am43c0&Y59Mo$s+na@?id7T z`y>f7)_f}3an3hXd}%H>CApZ5zKOuQmdAkj6bJAJ^42^`MHECVp5miQn-t z@(90}Z6rOK#C0-!EX-Hk4zIrnbM{&SJxKDDEGxe=-d}`Z@#BQOYZ!gfG*0ObYfj|S z&4gE>H~-K?T~wC#C)~7$c_aHzx&mz>F~OtdF=FD#WWjFBK~~kG^Fm|5&aL8S|1gS# z#V_b4k)()n6G!RWN~DXswbnO>AkKLvSVayy;EklzqJ&&)k&$h$GP*Yo*!n-VTEkbR?D}_zM>1RDt;Ydui6u8grWRX853r*zVgeD@R!6-t<_knxfpk zHP^A3(7tifz8=BvG(M35JPmgHAV~LYFl*Hs6TeTij%X&4JfLU`7zBrLA^z-Y;5%!{JOJ- zqc;P#G`iOwvuF7oiP{Z8)#a$^F@x?7#rlhgoRaq?4!BT_xq4Lu!xKvW~*ufSt#*r8`Ch+gIfMvUtT5^YVQg4o_fmU31 zW5SU51CA?lO-@laDVeOcrP({MR=hjs};gJSu0wqzz{SWERdRuWuw0iw5xgh z85P9Qz*49t@{}lg3W*k)+)*f+%jYUPdTi-tfsYc3LjhAx`hMzJgkIuHvROn>TPwGb zbDerMyP04!8L#j3xcS+YuGk6>%gHZYYD?62+vvhXce$>-R~LPyH@p#1a#+JKIDvK6 zvgcT4_Jz6kw1LZ<$-FE?;)iTazLcwAwGyx|aP|;~$nBxecrP?lU`MUYU2MYH5}`iZ zIZ%Opi0G}_-pzt@UR1co3MZ{k!f>gCQxKAZyO1cP1^kt9L@|v0sYFxop~e#Xq-?^> zS^mZnBLrj768y#3u_@uK!nxS7Nj=fF0LepxN-EULSgP2l5}f#QIyUKA$wb2i&)8Fd zoMi0DMP`iB^apN|xgebwCrPK|gr`gDhOa8AOuA-R=u4^Dd>$i8+yljSeEJ~iKP1wQ zS8xw(?8-HpNk6NUk`@kWl;saDirT#TY%H|3y?KXBnU_H1s(I3b?x~;-xKowwn%khM zgtz6Q+7uw#{)o@!t{#yQKwsiC9l5B8`&x?dF&E9bfNxdGDj7Wi_=oEP6gd-64;2>> z6AlZRb-!z-$D7pKzycmN3A4{zr+Q$+-u?p*gej(Zd(KLUONAST*cZ0WIfbI?ghkQ;AGk{v#-`HT6z)&$Mn%A3<2S-o;(XV+obW8AcDA7O+MssAv0?4DK2~FD&nlDhekA!- z>Tcc%i@{df|j?zF^^u1x0k|2*3b-LTMm~uZK$Q~p8 z$^@Za>=<*?!EHj#=EaP7GuzOwJcorVG|3C*7x6o8p@zx0-R zoZEDis=(|PM>@0&sd>R^ZAFyS{p;hN$JuOi`brQD2Mx$9J_Uk^O0bdf5EX&t+*PTb zGwK{``bO<|*1I2X)X1*X+E9TeWGbWOV65~%RE2i6NWy?mhC^jP-K3(FoJgOgsW}=((8F9QS7x1GrSZJXzv={X}?u zp{JsT?6c{z_zWNF5ykziEp`N4*%HCn8``xiY7~NP!eMTqk1+%rB&jP+Fpe`e754yzZ{ateAAZ*|ou?2k0 z)JTX3I|Y6?gFo>Rb}Rze!mDDuj`uBLcQDt_6DfwY1}U|=NQOG|*|uT0GNE?R8dYA5 z(|gjFs0!gSn69@NLINdM7ES(updWyhr23ga5VRt~?gIicNl0GxU#S&_|46OSw*OA8 zRH%ot45K}%+VW^n`v1MKIO0qZXQd3`AWDJIue4Pe1~mUgf~u!}+Sa7YQVNk!btX}e z?A;*gKM(XQwaD=Mp73}9WI&CW+yp+Cz@cj*$JbjN(S@!v1lp>QZO|S|9BvFXj3u3>WD8J!;H&>7!a45+# z+*etoUK-a%GPn%(QxA7ruI5FU!ttf#X);VRy*6?*qTil}myZZx@xAi=7Z$`;dnF?v zVe@INyBD8aLz=wvz){2@>U#`P{n-iZf-t&)$rWT9G{IP*`JlW~3P$qP_I z3rD^<^J-LF7g?M$_jPYkeopZ=PGGMj2MKuaBNoOL1`i>TgGS=r3tA<*4~$6xY@Gg~zNbK78I< z6L|Ec3vuykgL;*?wp9Yu*cy3XgJYS60XxI3;b6JY*QVv!x*(}IkOXONlLSh zthg*bAV|OvngGBFJ<@B}o|RDqtfe83C_k2v2XO0XUN}y67dy{ID7m}>_ckZX)O64l zKdHysQ_i6<#Pn*Nx~fg$#rKL_Dguv4;cCCEilj|2caKM09{vshop`CTJu))7P2=&k zfYNB}VKt7*3`Zn={rQDJ%iheaJna9g3oXLf@t<9217pAh(Bhii6tWn8*9eEpZ(w#X z%eGkZ;%dLu3Xj1NoM*z?z0Ey zDc7G8H0y0!wm&6kA0^!3QBA_te-{0{MT(?_9joWwcZ@6jy+z_$!?k4ERf^Pffp_@5 zLTYOI^^#&# z-4;NKG6rG*k|%BG`Q`Z=Aeo1WBeqomER&)pL_@uqj&`%I8YJL}nE3e3=fjga5}ob$ zwNSx$%uFx|+-lh+zw_@+>0wt&BbJVS9!a_dZdOp14!ap7_d#xahS2KX?&zf<)?G7A ztZbV3?*fc{n#+_MIQ`2} z5@>e-oCI-^->)H)6MzRf9!T%YWw?a%Bctaiq}}f-qSebeEKz5_O`a3a@8-(O*>qI1)4XIZKRaJ zFA`{sF2lk>3H?!bo10NmFWp7z*WEl|7g7#Y^~r@B6vi8 z9OQ(vPVpn1u4)NyKXXc&E-OPi&`c_UPDGu;;)7?BXo6=VxUz3@8Ke7~tRM&$28W7F zUF)Z{`A&D8yLX*|z!cZH;;;f{*^D|66zG1wq?}3_TIn_T&Y|b0Ji7|jFu8Jxn_kw^ zW3_8bTV7rV)thOxwT47FuDRewe~2J{kB*CXcoYH~IhG+VGl^vt z)fwAPi@K=SeL6|hnx9B^3fCy|V>OtEF_LtXAnoIfpOb+0WqP7T#>tDK$I6qZt6_Sw zlt|-a%9|+Bbj1pbs`-^q>SBIX0@gIR4IVylkfZJ~O_5Q$N*=EBV=+)l2(C>bus{z{ z!@UW~0F{y47k1i|>6h4;k16bn%l8-3FLWKID9>0`g1*oJ<9w6#PU z4(+2z^=?j2Ebv|pELsgIvueZ4-77sw8Tdq2^=pi3QgYxGXoSRBNPjYmP=v(QXm&RB z>rlPnB!DyyZHdX(YD21mp4e}(;{IfK{Y=7d;EzIrp8+3B{YEsq72Mg~w~sO@(znP=Mn(=;3NY}+ ztl5gfW&%c;hBtx*Q6c`rw4I_YcfvFAY|Y+0BmDxZ;2H}^8vK-7V0#O8g=X-^bVMx4 zY+Kg9dkNBJ3$2(=giZn0;{G2<37y9(>uU%NCC=yl6ha@vA$!!3!yyHQ&1j*&#Uj)b zLZ8bFVou^Yu-+h}>_|*ux!P>HL7KfNA0G|75Vp{1NJjUZatZk_Ic-HU@#N%J=5#Ch zBmrtS_zf2htbiTRQ~nU4Dh%dW(a`V`tc0I|4_ZId5L{3{1c@QL`XFTB7vo?UfP#Fb zo1N^_VKs9SCnA;nF4ul38FHWH?sO$1w2;)CV7&^nyTwzTbLry=wZ}cx8d` z34VBu_=(s55Fr=%%7*-YWr?4tMQfNsG6j7bf8w>{iS=ERMXuX-Hw--d^{+M}AZu(# zZddrzEGjRd+<4*7IH*&|1&&J;jc;pXhVMXhQ z1wBnY#)_VL94UXZ8`3C<895^73!G`OZ#W_r$Y)l>%NGU7*aBr_tUjuk@Yxk)>^{!I z%d;TMMPR07^XGs`FmO|Z>^TJ-fi#M5q&|s6J(oHp%bZH!W(fHmIySacCV&}p7=*$( zNPjcn`(C&P{0t`Fm*_2*e((d#fccvNFDc;2{Ql)+j2H$`SbfVj*g`hOneI{8X=vk4 zm1_XnDk~$4ys2IxJ+SUe+EQ=-_mfBlei}8PA~p=HU3$bR>r1^F>ISt^$!nT<#qnP- z^|e)4Phu_X>5!u$_tX<7 z=!G5x;Y+`l{1lB=WiAzS+$69*nEQ1M9iBFYDGt;kLjbhLEJ2bmQ@31m$5-@KL4n3N zA{IUTj!fW`+B;BA10Px!O-4E4xt!kH)pReXTK3IPGG5Xebdsu{-S^(k+vV~ zcLs;*NEB z3F?t^m#mZI!^*_`7=4J8deAqP(gKL0`LaawY}9deV2?DmTB)-5fP+?9BdpR~UF2+* z`66nrPo8`!v|C&HRFoLM@e+L*!B4RfUaFeKWJhsLT7#`Lh()J(#F^zQYenJA&w5DO zzPtl5AF7$>9K!qy4?WqCk`%4ZHrojGDzTMLu#{SoXW8xJAlDbQ(%;HUn02%QbR7}D z52{K}9;T^`Ob^|stoBkfMz-4E+K3)jM9z24Pcj`^{BHXB>OgzlqQZ3vJh^B?o1YmPoHYc?~HcG=I@*+gls2dT-$vG^(RnH zT;ufwvB~Rn4V^^VF^9Do8bBNYYdytDpcQJ94ky=#56`duQxb107Y{e@i1&z3!FeYB zQ(K36j97O_ViQk&Q?{%gfG3wJ5Kzo$-jk|ywHo`6sJ&m;pPf&jM3DW#xCqmCyTj^( zzZ3&M_I&*3cUbL* z-zy9b!n6M+gUMtTF5>h38y`+Pt{hbLfuLVeNSFwKClo$2n~Gdr4ngs$QI3*O8N?lN z7#>uHyA#gpmf|uJWCXY0C;LjAX$V)QCafEcg&stB5>)B9^%g7++m?xoXZ?FwX{kne znk+8^p}O|n{*OEEdtBxocR!-p8$$99%3Ot;NXO&Du^i9FJ1Znd>v%s;{8-P}Ozf#n zFitN>VsxZdn`|VC#Q`vq^;dUBL6pxAQ0bFHH+^75i&c;!-?pfH`axIX=|-t)V5+eA z3r58#ez+W|aBrOYSVguw^sg&S1jB=#*i;EsOrE(sB^HW>-FovL<2yxhi@O~!!(3SD z+phZYMNZ@S-}KC5laDjG3J;@AtP6Yp6b1`Y*mH&RK?alH$1^)GAIndh41O%$WiFUt zL{ZZ7?Bm(6g|ifw0H`nHU&w(kBpiZ&+892=^{FB{Ct~MZjPusNU`>|0BCF<396?1D z3S3p$8PZx<5SV-yn6T0>9LtCJ0!ofZ3>J>27OMEGCVh{j#&zAN07y3k9BCHBoRK4w zMC-80Qx5yERd+jcwl~Ebgf$MkICraEl0heZlAPKxigVn*OxfXbb@}mI*ww@G+iq z#Ly=P;=zUzCA<}9gEzxQ-&u_XvTR-uF<(7gmzMq^sqN@nY*KPMd&DG{1v`4cU&Zwj(DX+MgdeFKXcG7e3y1nlDUkcv1R5&j|yHt3-aU zFYhigfN)xm(y8-x!u#zeYy9~kBksfHPSjAQ%jie0wwuvqrTLBJ0#XIv5URm-x+ z2D3yC)?yD13dyg*4vMV3IIT%bSFEZ#!Z4+x>|0HVm3Gk)yhcz~yActR;8)`@G?P~u za4SVJi6ZkbeU)&yZh|t zIPB|0#C>@IWd7)ek00U#wzggP3W3=u9vj6=$H2$)d?dXn?SQfm9cBC$-TOYc_+rfMoeC-X7h?c6snKqfXLLG8l26emg;8iKvI%IVs%NW< zPO?zApkd3k*!p<%ON_|V6;?tpnab|_3&&WukLP|jhIOhJ=g&MD5=s3VNBJYNHD{ka zTgc`JLq~*Ind`-2=50&6_DY@sv)Y7TG#za;uKHikdVUV9uY<0DxtqE}Ir>Z#M{p82 zfTm!Ht5Gb*bFMmm(@4$#`Edyw$NAA7XS?UiNc0uoCv=5pTHDk^3>%~3m_VnEk#54R zoKcRLUxvpeWUMZnhkh-6If?4$#tYgqU_LF~TkH(B#p}aR@|S>MfJ0v|8zR^DhISa6 zPl2V-Hd9K)BL+RLu-NiMJ5*?acW$sMYoA-0)R3;bSLk*5i)*oOX!N6WmWY|+@ou_l zx|c?t0WPG*`|SRFIPY>4JeyjK0`6%J2wydC*)gH1t@)|KoMr~b;>4Opl#%4BLS99D zDq5^<{j*CKByK*jsvFswqu-HlD|_w(*!NHes!K{CP&riA@T_cS`UYiHGAwLo26qZ^ zBwdlKsr-|hS)3W0j=1!HgkCDyZP@6ncs=+8@rfdsW=~WIW);_d0B$%i{GecPb!l2s$fi`;uC z3c-2-mSj$8Q^p$l#DVuOrsml_1fSaGiwpUf_ewRB|=Qgp@)?L$MH z5T=Ys3RXhIjq>vXUl2!FrV;*|L|xKg2iOUFYEeZ)YGXX`I0#Hc+sgWP3BvYAGwBZD} z{e8S(tT_mHtTh5@M~Q_Vc&yYgyo6Nq$7n-==^aQ(xSBLz(43kQE-Bi0Nbcuujo%ar zxR^900v4Md>j!wO2#titcw**0h=?wT5S@sOLqL%cJ<`e^edAQw1O_=o4HNKLT(Pl5 zH&7YAno#;|lAz7W58(7`3K=6?^sW)A`l}N#=(DZ3orowVpaG?aLaWI^~AUIA# zI^5W^)%3_V%%+8;Vd!&M6v;0SEnnPNnVDPIxkrBb#n9he+NT%)N?EBKiD+n&0#65O z5>`sIvpRcqNk*Q?F7EqjBM>ZZ$&b5Hdo)mW?+{81Agtn}3slCgkH3|1ZnRW`z&gNF z%YoJ;5Vax(UhD({7Fw=s)8dnvdV|11t{0_0x`W_wZGY^~>2?#%Kf0Jqp+)zPF8&DA z#oI^RN;!~1vcLNn9M`-DI_6%#7=2@pMTyil3h3ImRpaA z6jx;A$?Y@NtaS@k6%ggOdzDze$x#?-eDkHTWlK}Hq-idvuc|FKXn+{$t&sH5F$o`z zAL%Ha(5P^{kM(YJ?%S4(e>$$)`|Jm4lsw9(#nHCu!cDc3qZ#~5jUh(3f_g9Iou$(7 zi1YiMO<5z{Z)qEJm=r51h6;)6sVF)*F>VaM@s1T)0cnBt!>Rn-l9eJTCQSz!a|C|T z34T#qrq!}@n0~c7KBza=s`L~aBL_1>5XDc+Pi&)HF_EuvB0r+GoAbDHmx+ZJqv77^ z1l?kCLG0ov4=R)1`O8P1N%4!jyIJnM0Jhmd#Ps%{$G;`3g4S=zigy@R$Z%zwA2rY0 zXpCoW@bEOCnFI?wG*i>1;>2Q7rLU65z!pdbaQ~660(@BtKXBiua-p~k3}ADn2Y3&C zz0*mYBd0_YnWMFYmIO^QCgr_wUannU`D$=zuOGUJoO! zI3OjS$99Cu%bYmeQ+a@Ov(EeU^y>8v4Bx)rKfk_o`A_LB@_Vpj;YA_>{6D|z-InvO zenbqO5O|-rct2gL^MU)3d8amBrrv@8vt@ej*>(p`93tq8i!5ig8F;#dc|4>Iokj5A z2CwUAqCV;`*^N?1g%T1{Dchk7uu|aP75+bq*%$86Um3r}>^S8AE@q2q_I>~VYT4EC zlYePh*gsk}%|3<*7Pkv5Mh0-|slG4D#T##Oc1ZCvxZL3m%vWEf_B@!}_DeK@ zZ6B)<4005a&3@5wp_h2(j~Rp;1!_kSb>NL$*W)Pe!n_2`5anGRu;@Zf8%J8HICRtR zcff@&?=kLCiG@jX(`U08H!E>u!E-4^a^$$ND?#((Vtb6k^VL;LK>~5Ef2mxO4JpsR z7QbMLJ9HqZ^&ar4GVN;te6%T2^h!_gADAHLIofX!nwSIUMkHKtnkxecHXkEd<#cp4 zHZ7o4Ll6+}ce56G5DJw%V9Fy4?EM22h{kIEwe*E6*k2T6H(XOrabfxA^7n%mEyZhb zMbCyc{Er}-}b zY=7t3b^KQ23<(cha*>Xoe)(i?3E zK3fi&3Q$YM%cFcO#z9%INzwGi$H!aB^KYdyU{C8l+gjhE$AXV??kVnqds<5pccQo+Li|G`oz&XvO5)`(gVqAYoAY`+klEz1@5$Jp(sd_Au zIAgQb*K=jZEE>6Qaa5`}+&U247&5$<)R?Ar?x>5mxQYm&E1$l`rk^S>Ywg*0wM@fT zj;%bmM#hY$+MVq5={p9aU;2_nq0o&~LtoTz#Ce+jLPF=AeS1LM!@3?OJb=NaXLIi}=QkwXdAW6>b2H9`fP#_z zqS}x5zze2T?0I7AYzdVvIg#GNYuzMH#I1b6E6`qKw%$=$#LkiGVCyy`WJFYQ`yD-J z-Cd1qK5+fIo{gE}-d>Hns+VOGsu(q<3U?Qd*L4A0I<^iU3a_K}7WWMZzCQx5F+9C4 zwVK&qeLrY?3UgC{GsdD$=!y|*JSaP zFxRbuR2~NC}mKVNx)w0};sv_ZI7~HayHU4jS859WF=F&0Z9a53Z(_rLx{(ckD*A&H3mLUYi|oz}p` z*whH)wSzeNc_TepYbn4gmF2@I9*I1py=sSBdr5R1sFpt`x)t+6MQw>WNKxiA$oQw> zLXdH;eZO(6+rg_2nw8h{sVQp<!KDbzPfv_^l0CDal%t@&xCoy_F&W##zg!N+0Wbz&ls(deRVBpJyRL;YP*>`Qq*& zkm6F7N-L!e8%vW|pkK-Z5IiIT$4)gYpd-ixFh?_;5U%_iA(Qu6-RI>2UaA!s9Oz!D zycTsPR^3IQSuNE;ycX58)Ex&M7LRgLGIem;^)M21mL^5we%RbqrSce50*P6itlL!N#RU_QpW>S~_LCswJ!yA$=jekg%Gw zpkwv*=itSO5%eEZEF#0qxibIm2)`N*R{_%&e!Vzmuuaha&PA)&ex{? zJl|$(<2F9c$*pf}%B|Zh4O7U^xIRDV`E{QlU380+84q4a2%Ew;wCcS?nujF0!Y?)B ztDtU3g*CD*9pA9C_iE<`o052#Sq2P>ng0Hw-do%dQ7x5Y86Hu+{x0D!cF}uM44;un zf?@xB88suXLN7tIL=T|PFp>+KrLQ9y8<2psh|SYy zW##YxWBRgB%fOU!=XDkVfuGR8gwYvbV6w<+YIzDjIId7$R>PuflAh1%9Hbb_)4-z4 zK2(T<6E5z5Wy)9xa5yeO)>Oyw5VPukOwO@AL}9v*7GIikwI@UF0!IoI>wf&23^M9T zaYpp+P9+OiIDkQ@TnOYDyi@G6BF6RX*1ay2RblkHmItV+CJc5KQ>E z<@qMqM{y4(HWH9(ORBfG% zaiUM1D6f8?G^SXzK>N;?v<$6PB-po3I7sq3$Th2$WVi~i>8LDt>#}N0Xh*8xIVdq+ z(tXO}bjbn$eRLC(wz>o4wiNl8;ix8sf`F}N;L8s7U}*!2`Ye)BgZ?(9Mz{JGu>MNo zsUx3VKg|J^&sD`uBW&_=Vwz5dTEeocH(Rw{ zu&uSG?SF~TAkNjG@{@jpbeE29bWS12wG?TJ$unXw3VQqQG1>Pg9Ks|LTL%{Schawh zlH(}Ab6e&G9+aPQEq|60hpG0_B#z^#mW374-m23@G=jgoVc$B|!R6W~p0yb5wn6O7 z{&cC=qyI<%HXg71!cwVuBoK;jS%&dPvJYkho3j4iMz4$yB26>jrGJ#%gS!VvSE(9M ziVY&UBa<1JBJ78!-EKBw$DClrv?M$Fdc^6teWkJr6kSkWqNf#N;PxlpZiI;F)F`0! zw9JzHXy|J<>lxbSkME}W+=?E|_Sdp`nGRs_FXjI6FXjGXzu%;R*C;Y7e^(4$W zn<2Hczb}Swd1?UvAHp5SNLco>HvXSoGbwb4_xD4=ahc?UycuzZy`Nvb-ZpyH zv-ywtKZYZUL;3d>HR%an%d8*;cjdr~0LB>WnUY1CnxGA+q}yZIC5 z6L*0BYo}*3zaa4B805D-_@}tOuN&(d;{Hi}M;n}#3yKJJhaW8NnR?>4o zv_#ROfwA^719s}#jx;TX-_Z*2cCyMP2tjM(tivL3ygW zDVS>=w}f#*9=$Y@3)~ux*&8C^?x&S91uDnqjLw3RDdN{JQLW-@dE+setB|45-(^S1Q{p#iy88u1d}6k#KO_D>l)Yn+rQf>k znYL})wyIK{5w zs@uZRG;U4eC|U@UPB~lMD%f|9sQd_;9J>YrV+FyvkK@j3mg98_;RU7(OgXpVWR!>e zKf0ZA+=Kb-$3MD#YwQoVH_76HL7f-n07{x{XZO9lU$=`g@;v&oTVxnvC`iTFTKMoC z3G+m_mZ3&^B$qN)B5s9zT!83-wzA*FQY2Ac?Oi$lnB3@=)ldo1CEUMVYdM6&A1XUV zx4NUMY@+?(cE-#ewV{**dy}Lb4aZd4tMj?N1T8W;V_gAHtC6#QY%2|_)P{~qq=2V}lhs}c=#HgR zFWBOdVpZg)^qI68u;^mlZ&+*|esoW1_4JcbRFOt;z+|-0kVWGOQa3KJEP9SiA!(h? zO+Rv}2sq`8<>%}vvg^#sOb2wuj9w(LWT?D?no=$AJtJjU6=Jh%SB-ZLLTGK>R_*1+ zJ2_MSY4MZJ3w;8|Q^m9!{c7X$2k{B*#G!Q_$VV-bf&S4X#B;91@GZw@wf-Z=S#bU> z$8oRz2RUv}+4QK&+eGnjyY)KVxg0fO(f5IIKy`;w*&(|eaxtOOmBZd<8?K|4O%l6L zp4ssX(XP+r)(Q*37}GcEc~j?dhj1&`-%3HJ!XcQnS)|F?o z6UT$BlcQt}){An|JuHXJD`#&5r(E{)%6+p(Z>oSpbOlyRlK=&7-RnY_+4phvOtf22 zi&I9{Omd*m<@CChVlqN1z486?4Nj10d*=)o)mHLjZ~ITel|w)On$kh)g!TY<1s^Vv z8Q>!Gq0U1#{^2}5!F+{esaz{H8bz;aJw47CLQvd_f${t{dB5^lKf2`*g@1t5#uE`Q zDlJ85LW(Fzf2x^(gq=_4sE26i@&b{4Cd7(5Ut8EMf+P@hGPt4aOCHm4*VN zw@B*}Z}#EYCsXv4sEMB2b%EPogj|EoUh)~s(~dfcvY$2-sa#_E@qy2zObV0@DCyT% z3g#@~5N)huHL0Frf0X;)<8m9sh2}^r2TmvX!{@o3L8qnlBA~Cula*Z?;z>SKyCm#*vhMqK?v14mHan{S>hMsVcao4TB zRY`5Wo|%r%YrpsA?#=*bl+aN8`nR3qcJyDPVk5I+CfPBg(kR}kM5Mb+-=M#2Y-@g7Ar?lM%s3wE#%1+#3)ZLZvavNS;$Sur&Q%LEJ$E$Lgs z54PcG2|Eu-NE%uFYs;b~yeNB6w1yLD6=46HX$is+sSHUI3-^E2uJz;foWpSP9pHmK zU*F&m_=|3MV(*jt&w$Hs1f{l9*)ao30vBZ)a)9%9*`M2VUz(Y_5g{p1diVymnYzH) z`M_j{a-N}-Z|TZA@9H6@)r|gohr(^Z?tU^vPpC&&#TdyfdWJ|=hbgo^`Q^toRR#pR z`aQJ4RaU3Nm*?l0y zuW#+!PX3Sf-MIH_W{}A!)6Yz7OdRK-wzbe4+5Q0J z{Kug*4ZEFxN7QX!IxUeJiv@+z5|2N~pLv=G!f7Ve%B-Gmuelr@ReTg==naL)1-*4g zrCGKG1r}54@T}j^Josm!K~imK5bJ++n?*ZSs@xq|=E_Ldajf>T5mTO&W|BnrH?aeh ze(7@#>~R?DP!jC_8?j(H`Ts;LqH_nVnjyN1=P*sQRM9mWaCB>(mTEM0*i7<C)D)zD zo6YNP96q_pr6(KaGOPG);>_IqU5WRY)QdY;a8{G(jI_xIW`Sg9U82)Dy`u|K+{OkgX8eP3r)mBlNG~Bd4a|iC-E8`TeTCP2g zh2BDi%gcsjJ{3g{N_Rc5(XmCitgp27%tbw2{YShWVrXRiOT5aUW7^xa zqX-Rc6Wj?D$Tq;>8pKVml^s?B3IlDT)=Et>^wNpUQ8X<;J!pTR$*Ohf;Sk^AHnc4{ z77q4%Y*x!NJ$d5wmQpGZee3>)DcZ!dNco*F6JI%MA^v4FpGNn->Yu}*9%FUn8Hbx) z=XD9&k&izl2F=}G&}yEb1zhyBj0!XYu^o z4q&M?X?v*oMoH3jS)%VbQI^^A@xWI4{+D*4wxrXTzZ0%6{Hk}Pguo>uvgust}h=xU{v0JKC$x(2v}w?%wGV>Ot&T7omaQT1%f=#&#P=_ zC;esaqO_oUQ*0c=Yt#}a^YD*jeVeYZW%=z`%jH#^Bx=_=cs92g>)eGCs^HU-UH9=o z^rIYOpV#$`IVb7oThFYlDC1gQu&*X-zP!{lZ9R~Ff5~d=m>u3Y*pYx1RVA2%iZSqS%1ORyqV^Azj;PUoTAf|zQL7$1!x9x770(c!Y| z5oeEV@MLgC=qTN{S-dDI>Z~)7cNEmw3{*Kz(b|t$A3@~2?XE=I@|a$CBcC-NV|Wgh zSwhqd7o`-@saJX5P%Sxe;__8{eX}P=9f% zKB;{$l6oo8g2Q<+-zo5qfe!MI*@^2F?}snkf^@){b- z*}MBulWf7pY?RC*M_6FR=b&sp{i$qDe45htW#j|y%fH>Tg{5}19qUd>hb9w~}%_S4`DrtvD`RtCR z@2hG#x#FWSkmV(hy&KcAlL{_hJQ(8T&(aW&^?Hjb`1bSFR27;&k4c(Fk@7VageaZC z6_v<;EhL6UYNbTpmM-Y~R)%o2g<+8fqSN~XaI}kEbkvweTGYg>P3T5i5~bJcSo+qq z&z4#mp_>)!n0wTF1h#rBFXen}7)grH#VP5CF~69l95S*J_>TgI%vMreK zmK@|@tK?f>M`Y;crtlG!`fD69m3rhLH$4c7*u{D9HNQ9KxoL8Ej?mCY=)p*jvvAZ6 zWM%wK5x3a|Syg5goe?+EP!-S%<}yRa6!|Ws*x@3z9#x|-xMbI(Ax5x?QeLh7GJzcs z!AGze!2=H9;;o_&3~hly58)CcPz7u*40E+P_+H29gXvvdA=}=!2!<{(9z!T5xFa3J zauJj%MP#a*u^0?5xE2WdOK1k_j+xTA;^`fc3ZNgwat6$Q`dW!IC-czf4L{kNJ80)0 zNvpvhP0v*xH;5einVX>gQcNR7eReX=uVXC#4Ub(1>1dcgT2zNf-o!K%8e$Ro%j4Mq zFU{Qa@2{0qP}E68=>bUcy>`kTNOqP^O!KA_!?`?MH;@zU^<+k{Jgv_&IqwR z9Rq2cp})1MBdKgod{+a1<60kk)!7>YhsyF`hsqR$tt)WYzm4+D-<3GuK#4O%w|49H zCO-o1>GAy0Ou&_e(EjoL!h~Vehu4#8O}I*)g%SF=5^lM=h*gE*5RGYtAufgtEol@txVp5dW>V4oV$Xq%{7Q|5r^FCry9;oA7Qglm1J4-*i6Sk z0)nvAezUrz{fjPSSb3muxe%N__y$1S9y+0BK!E({p62II_JhB z6%+3fdDt-Qkt#zZ7-!N!7*On&njRsQ|FWxTt{le4ew9aCTRBF03V><+7tNny1i2dP zj6ZUW3X|5~uDb(+mlu0AQHXltg6xlq$-Q$_x#X%?YTbmD20OZ*(2N5>qoVv{mHwN-29G7pE2PG)-bzeT!kY+aS1XD#fn zF|C>?Z44jhTCP}(ci4{8U0=D<3c=v-|6-ir?<8^Aq`#2dPOjw7Zt1E-`h={s#id?- zpRZ`P(C(6m8nIG{b#1wd821*3x$E-mJ!vJ0n{}jr%nbNWsJRn+qei6=`zE$kI~iRT z)zmjxq$@;6PA-R+KTms~NCylm$;JqoU2z8Nyf;U65SElld{U-U;o$?9Y!1|MIV5>1 zv3A$AoIgp55)sS1G5)0|uUz?KrF#dlULR$Qz!u-$h~6#}fLO+{aDPZJK|9vD(b2Gc z-2T30V?jItp+e#x3VYT&dvAt$fn+a&Lxk~* zSM2|rX!3|dMcC^snd$`lxW7Vw*w8Ernz?>{hb)NuXx_fY1YCaak@)}H770NN@*CCZ z{w+2h_+LStypo-7-dZE}VP*5L3s^@KR5LXC*SzK95S<nj9OnDy(L zB=BzLf8q1`;W`W8l*%xL{2|7{Kot1fK^$+WZsI^D2!=v4F9oSa3`+jgrrJQkxu8W0 z&ZnYWldY+l__mY(9Bgp*8xlyjaXhm0E!LJEgM&qs>r##4dm^Om@b@B}xg#a$qUGUR zc5*jNH+5b5%aUz6EQ0_`L08~4t5RLvxpH0c5^sr0yT$fk8|u9FDiVJ8bh|AXP-Uhd zM-;@>WSvZ>SyOYSoL!Ksb+?DLiw!>o$oA~QIdj!f+q;Qp<Irpc(`d#yaiQS9Ub%?Is}It}zAT)M=mfn;t!T^)@s#mOU;n;)gZ@)*sOlk71}dXJ7oHNnEN1@e^i$I7Gyj9SMft~(xp+5GUr;74F3 z)F#6SB~39uc~4(ysbtS-$%UJWJrdrtDT=rrSnTMaA(u$6i4ThqIenJb#B$!0?_0p4 zC6`HO&X1kUc4IKnqJ1VuZT^MEEh(yyCQ#^8vQMS;f+DJ|zc&NBrK6AQdsB)!+^lXW zcrxSk6%tMA`X}$5XCaGlacUR0l;K?d$K;0~R#f;s;D_&Z&ji2~ne80Vu#8D0bfoll zTpDv>Q|Oh*a3vQv89P&bVZ=yrp0r_8q5AR}tZP6C;Pk;a=p=&iD zYOCA1(^ui0(iss$NB7D-af~C0DNs_ z&jMUgNYKSKSpG_Swq@_}Ka8_|%B7~9yy85@mAxM4A&i+vz?Fg>(w2MD#irmKW-M?l zVM}ZN%z5Yjlam+X5)rraqGB`!5BI@yXU`tkdh^Ug0HXqC;l|V`iT_(?LyFT;`ehR`A)tTt>Q<*IZYCvO?rP&L4S zKfHs#3VwET&-}tpzh}z{E_+%$w*c?KHV?re)&9&0FJQ&mrwq$sd#bUbyK@BLCy#JY z-SVvUoW9pMqvooDDeaIH=EI;Bs6`o6o6uham(#UiJ1vY-p1gtN+TEh2A~ z`QA-?GL07xEy+tey$7zw5x;&99jTQz=&j#X1y`!_I%B`?aC4kq=4;XZUUP;!%aNzVtQ#*y-3B6hRRcmEKm^?&0{tS=j*#@-yZC8q zAnQ?`yzTnpM>A_$L~T^WY8B~cf2T~E4~b(Z;Tf($<%f8ey|@ggMV?w8&H9bo7e8y|KC>7Vq)O8%pf-Gr-823+^-E;`$o>5wN3g zzV!XLN`3-_gOgC4lC4`!ds7|dRCy`0`0Xb`uP%#(XkGJdi-wKzXwU6YH&xO-j~vNo z`gB1TXQ`r2$zH(#$zTvtp|U%VC>AZwkcC3Ee@`VF4WAXtMTC>NHAzj1XytG>NXwB? zF?ZX_U34$1n3i(;LD;=tOmeXiOZuj`-e8ZwC0UfNUITQ9!%4ym|4~E#5)xU{FcOFv zkI6dDKzc@5oci5In7+Gsx1aTvso|SA{Q+G*rXgrwS>p0O$F#2D1U5Pe7;U5ClwYb@ zamNx{d<%AUmVInID5Nc08K)4ykH^Q<$O#Rn%l9RRlk+UV%4iR*U`_#ljyw=|*jDfc zK2C(qt_#I5G_a0h86^YD1?|fT<1CQO;r3(3?30Vp`^E2+IX#P5AJMQP^OQXkO(v)D zuF*wfOi||qLZiYFG++j+BC7tUX9O|guVqd4MUq_5nFxZgP&qdhhK~Dkc z2et)PiwDasMPEbJW=M~Ab!;j+YOI)Ul?T3`la={k>SGeV$b5Ida424WK4x63xev`5 zkF+a>)v+o<=b$tw%g1V3KmFysUH<+yg97l5e>$8$>0AreuR#Fd!y#aPbWeUwZw40L z=B!z`sR{YlhSflT9HE~3<$^0(?Z4e*k2UY>+sr!-RLyBGq`(ZMFgmDmlDo_<)@nDs zj`?jEuB-I5YJoV)0goMOdC>b+C7KwiFFTmA#=l^roq*`_dmyFSZU@WDXPj^V83+_C z>KmZy()#y%K~xeOS~!5B5nlZiWQE<@ZPyPO@lSfGzEHf9`$ic|t-SeN1P4e`-BGB;N2(@o(ijzBbrRvpAi1>|iL8pt9#dz-f@7XL94$OQaL<=fijBsiV!t!a-;^PNJ!}NyY`v zQcS`d?JyxIra66&IfhDgh1qFpEmvFScLecY+EyO zluLSj`)AFS2Yf%h@4wcw0y#L_=$rvcXZ&4mkjm}-MebkZ@4Sc2;{2Hr83Y5|jeQ(q z-p9ks_jDU<3m<<@@G&>kd+~d-<(a;!pe(u>dqMOVD+bRt#kM0QO@QiQv zCrP9>dU95ZbGXmHH~EYaH{bo~!zS6|cV?MS{Sv6Uh>MMuZdBp$Ut*=w`Wi@aIjo?v zzdNkHqOsxQi<>8oItE}aA)q+$iF_Cmh<|usfaomB)%``zK|8nVCYc5qlwQpQ%;@9I zP!lr5(3l6yv%`%k5B9-%lSv!{!xkLf-}iE2)`JY|>jjO3ssHY-p2#e$G=Q}3j=TsB zMBQUgK4gGkU{uO<0w5P3L~?>5BVHT_!4?-0Zpxud2b-scVYjJ5J#k8>Ou%P(%1c#y zO8UTUIl*iFSln@gnwmi;1pI*ZIMf;Sje-2^f@Jb<>N!4H>;Wd|!23>{d9a^M#~;bZ zbQfjOeXbx$)cdUZa=%DfUNs8aOCQQ63%>HagTA+}#dRNaMe$1yF_5q_Lk86Mab|dY zsV_mn!RGueDcJ_m3D3R`1_AMgK05qu*4^(o@;li3dr<0yvB1b*j$?P2X%#}2z?A^j zh4)SI`jctwY%$+gV^ykkZeul*MQ5fIQ+6P~$FGM?t()^)$D8%zsw9hg^}qqQ=-dFT z7lpUs32IF2GCQQeL-0;%wL@cyu!uJRqetB>w^Jbx*3-LwwICR1akN9VAdu&$Cd;p? zo-Q>p_SQS%W1td)(!KnYfO?7C{ejEaK*0Pf5oH!$Js2gyTq0 z$a0Pkp6|=c)4MAGvI8XfF$Ht&rRsZa{kTE19xh`zh&;f41^{o`3c5A}s`s~F)iUm@ zyWi`p^U!>=c2CuECb|qWd>TQYoS%l(d<0+t@B8J)F%5+6+T_EYo3@^H2F>l;SAJXfz^P1N(=3bODIc7( zfezadh2mK0RRHg^O#LYk!3FMZ@TrPHej?|mZo5gn8vtSIJK&0f?h=rP8X8I8waL(O zEjl&$zWk z)nwjoJSF-#(q)LRNuR)zFS7K0}%Uu4Vh??tPE=Y0f%g zL<+d>kNE*C*X2jsh1Pv8X~)#42#Z@zhuQ1qh!K5FCqKqEi>ExDG2ho;7+y0CRPHt|>Hge*6;zqE>H@g?I@!td0hl*%)1J4l$P)Y0`=FyBR*FX-GTW!P zy+`jFMT9l*7R;*{>qaf(3M0V|r+{O6MeKav({E*#MhzV^O2-}DT~3|5r>UqTuo>uimFrYb zNrDCmY!+ql%GcJX@ue)xTDdbRQ#~+I&e$sFz5;bea)JTf53;@SvF(Kr@(%eMrEE@X zH3KUB`Q&o(bIK|o=jO*lexn`zK-;)6`e(p2D@^&NxfO@%l+>cmIOUmMy++5t_#pi| zPUk_fid_~5xJU{}ZKei{pOD*MWb;B6)kbV~%N&@iarS7XNW=PfIoCB;4O5(tgX*olC@#!yzD>_Y*3dwl)FQ8$S@B<6 zNml6N-b^)BYWsMJRIs-eSAi4q7lk6LFb>b#`C9%}_VO3I#0nIL=Bw=uDpyu3+HRha zMisGN$@Egq-coF|KD%NrS)q^y z=1M}3_od=mqLI)d1$rBEDwrCePIU+v8py+}9aMX+G@!R~U*^FPS)9=`lh0JheGm5V z35Dys%IY)zOa(Yp2m~ocNJ@ag&NvkOZE1`PGpE7B#t$+&z zROr1_!Z*$loa1J2bl#2CEt14Op_VWq8f>CiN(*=+Q|$MCM4*ZrfccZu_O}vDHOc67 zjFpF4l@qs78r+di4Xv53T2W#g(=F;}vQdX3YD!r0(|TRhmS*ll>p5+kYBiZNXskBZYA|g1FV&0erPP`_Iw{US-{z+Li)&(3 zxO$NTw(7b4b=>gyL9ahrqVZO^76{zn1x}*v@8B;+;P`AyEuay>br~NT=csQI0zFAC zYguGxdHy?ds?DxJKk#=v%=eCpAO-_JJRpK`|4Y{AOd^@9UJ;2-jJ9X*Abh^(a;SQD6!5lqj)i*gH+U4 zCTvpEDy-8mWrI!8#WUeDn#SleoE|Ha)f;5JgLChLQ5diy5rqoln{Qe->6Ejj=5a*6 ztmdZQgm>DCS1-#9FYE&XoCn#;IKlh3a5UKmM~l-;mjBedTovHxm2dkmDA4k|N1aRS z!(n|}#u)$>%Rn>^ix)DN}Qr5|IrX|77oTs#<^gZ%m2VB+*~l-G}eh;PxNKIInQMmXce~{TdC%; z&}8V-uU47ynRB0`zo9VDfPD9BpXH-`%1Xmlrt<0amyv3pbyFwO5-!^;s_ry<0DA|Q zHVho=C2NJ+z&JQv*sNw(XM;s!4By$Z8|msZL#{9M%gDltNz?JX;;-&0V|OJcYVx{s z$*-TSuglz@Th^_g=1Wb8Lg^l!tVkrjko<-NI<#x3s#Y6qj{VI>Rk`7;Yc=)fK-NvP z$zMe)rrC=9u@g0o=N^Vn>oMs{NJpVw^f|pdFB|bt3aHEhb`~D8G+755B<4E8-1_iW zb&9!mx8oUvZpS_9r+G1^5kfhw$lxhIJW5Yiltz&5DUKFe>i24w~bzJe0c&2YLH%V z+5_9Vl2VEH`~_QDgFtxQV;$4}dK0dCK6qni>+)onKJF zU&tO720fFGGmZ>f$L&{2o_B)b3dakmMjf9lZcCwIzmRhWk8|Qy4lm>a5Id@?wBqm7Rsx zc-Z5#358#6qm1Nm;rS+BfdQmv+gJfOaxefwPOSt8f?b6s7fg^{@B~uqE*8v(_bZ$D{M+xkRy&@X*n@jMQIMzNcD7s{GH=?L~f z(6ONvx5cK-`~0Po*si2~gz#_Io89Bv^?nfHC9HV$uZ^DmgmeIH0b7GP6GT)h6Seh9 zxw6qSXM&*?57y8XGjlVxqK#-{^nDHg5!osUj<0>*=++q&G6@D?=-e%qySqQv5iuS}s zMk-0RFWNf{PI_=^UfF&ymI5eKxl-A@pu1!N*NEY7zD_l+`VRn|K=BFyuGM=`*d8zW zW{ew7F2h@B(ePRRJz?T(Q$tkK9%N&k!Wy;Y0q+k&6|e&y;KW3JF>+y>J%C z-Q)1uy!*@!KR~BQR>`4w!WI~2NGGQ3B8T@A3|Rh-RQbtZhvRAZwcH_JoAUo&xY~<2 zu&Khdhr9E#cmAV+vdfAiOF1H8nAomzH=l=j0K#(o7~*P?FE867B=RA3^fDB*;;CSA zM-wi)BwVuAp^{#=*HoLSz--gXbhjD8`5KCNEjtH=5xe|>8J#bz=?g)KbYHV0G@)?j z1p@|8ox~lXKr`pF2jpPquie~^e%(eGfJrFaR6V)~>pt2imkO3*50P_}DjonYG-cA+ zHv#4R(-h9%&xi%I+8e-rPX!7W?yerlO}6t;d7zT5w)0-Ku@z_o_Ez`z8kB*o_3XRt zItx!IVi1mJcT6@}LRG~^-8F&Lg@9t1h+p@(cihOXrn|ej8@wm6JZMhmXcWom>&HfU zOk((TZN$qrfgyE`WzCS|zzW2z(V4z#IO1DUKf__z6-nG0c^i{wEX+KO$sMNC)Iq&o z)EsAh!Bb7grUQ`ZS1_BwkMw9wNlnC5d>pIs_~gVPxVT1N@_jG(^sfJEk3fQPzFw~+ zMEU^-bMT9j<;Ji!MvtnouxtAN5x1+g6Z;`DGK|e?74X7lQ(a?z(wl+`$+-RaT&D>a;o7_&_biWr0^nAnbp7Jbwy>#gJu86cwIwz;eWDq@vaV6 zD@3%<)e)+zYP(R3^ClY4m5fSox;xu_iLXY<{Gg?5Ed{rc@LtW{h1;Qcc3V_?{>5aW#{cog^!H>*+jNHb$z>Xum$}U1&Sz}WfiV#)b zGe5mQw=-~Wv`5`|octpJ)m{$OLiAzcs z%{gsooC820IpDP+MecLk#Itj;(O&Rwb$XD;>>^VLF)=Z9#3+F=1q?ajJLd))FuRF1 z^8T_0^upcBwj4>Iuhq<_c<&Pd*N(Sqq^2d?^RDX~sodAXwqljL;NA-n)TKdO{i|3( zvg@yUX1uLU#pd5(bO{h~>M-krsAunXCOln6XNP}S9wCO`JV36Q1S>%wcj zRbb48_tZxLwJkR8;>^sBOyUFInEuwg_m?Vm=OKjG{nALnagfpDI+m&fSfl<)9coA* z@nb5ml&CWlRIQ#mY>bs)n*hzj>wprhhuP8mqO%`yj3;R=ri<`pliaRKu}-#D3K*M zXqgIu?O8q!&{d#OcS^e;oM@tgnRO=aTzMpZk>*6Mr|}dx?1-j~Pvlk?>;)I)pjGeDWjwa`k zDQwJkG<$mEM}?7fZ$iagi4|@uyy#!#myRfF+0+M&F5)K}ysu_m?DHPJ$kpPrcjQKp_le|cfVExsF%g-oqo$${U+fjtuu(FN{d`WQJ)#^A`(VG|*ZD{2<2u>p3F!k#098y4VTY$kyT&}s@ z!BS%7HEj~NN}3T#r{h2Z9bKB9t*6-*%(11r=%@)^-_@)!m8vh&;SK#n+DDDGSetuY z?LUy6v1(%CeXgMyLEl*fd^Epb`VyY}%#%db=Pk{l;YWeJvb1kQO0c`Z`V^_(M-a8WS+vn2bXDkEsOp2=D+f=>v|wLQ>P z35DsI@0bz*7vgoJaADZEX#s_3yoU~);91d+_zO_`T)p{wee8>CYuFvVWa=tBNwV(1 zYG38Jd+zV%B=_$#R7$hz?X$BKToPhvc8MAwbFFm=y?0;WollB02FaZOk>t>Epn%s2 zI0#S#YH;l0!TswKfAk>!681`&p}MHk5#Kt$#5-#ZF1w`Ye-{IP?=3fM&rZ( z^eV&LnluFc&?|wh?nzU{u%v~R*li({6b>e-2W~hJlh4@6x0+^vrE7^bLT;N@cJ>|4K*#4;Gft>L<0#f{VCN>tVF|x3&8yBYR zT%G^Rxf+(-FX%d!!AWh?HHg7N{28bC(<*RJ1#L@jzDxg;(QMOtpm2^P*D|j2bJ`bwR0R@C{0w?DZdm|ZZ)=_AIZ^r zN5ZHsD+wCdQ5w^yj{k3PdUD>Y9%?L-96b;SDJ~}!wneGvOi~D z4JQk+d1T|1e6S{@ct?1tLa;(elr_L-GTamaT?q~(e-RTxRFX~l!swT}wK{Fh4x3BW zhh?Pgm1L$EfNN49r zx|}`tO5}Xb+7wUo`PWS@8NPGWYo?peA4#l?Rxa4JG8l?o`8}pzekJc!iEwVUN>hPT zh%q2ibP>>@p)K6h{C4XJlL$L_M?qrtr_(7u4+)cnMNMYeE=79{aEaUbBEZ9!$kS79 zsu^g`FEX-IOpUDz++ElFveEj2aL2s#kh6BYEhWJdq`_NDu5IwMg%*-l`5hvLBk<|(<8iTQ8 z!x1+RJkMenKk4-l3p407L_b{zI38OP8QoSF=zrkNC06x71m{iXn_-dyv7uFhhAT(s z`ueoE-~aeUq)OSnalykzbxA|zO)A0;U)*-H*UbDV-T%YuCtd6$zMOUts6@GbaAyrv zGFW9;wn!HJ<1ONw_s!2Y+rOAPE`oJ5+O%1u^u({cbw2%M7;RtekE5NAqXF+4%=hp~ z!|exnK3_=smoYJ@RBhJYKF1CsI_AD#2YsHQziXQWx`Xh(eqz0UVEh;#VtBiq6&d-d z>INcUp?k$L^6#qz(9L zSO1t3e`D9ykp~)5AAJp9s z3g2)m0C`fOd48)=P*g7%Zki=sx#CPwlgQaH&^@yL zfrnX#2%7r^!I)ZbbCmDJ)NdgOXS$M@K(}1XhA_Y)zy#O#49hSjL0HVCrtx}o10zbe zF@U>omhe~ftt;XoXJvtXJ9yb*A?4uILDSDx%g@aFCuv&NP&fsKVuQh8r)&%tv0@8j z!H8DsCN|;>hcT$wN)HH+SK)>ntCv#a^H0f!#AQWYDR=rq#2;g75PbN-k;_$YpJ;cs zq5Jw>eu>j)3x+<+{})~F7$r#?Yzw#D)3$Bfwr$&*#OG2 zSLVOWm6cCbW>oAQJ9Z3pQBhD5skPD^xNK>a#}f!Y{Ay3h*VYox6)0CdWzpR+waTRs zt9<;Rz1k7}6&9Zlc!Cb=jStiLvnNjX$3Rn6=F?8bn=Y7K_-f-3U$zg zd(P4nXd4Qyp5j@F8 zXaY_ot*tYWnTVB)_pdQY$y=U0`eT@%fc!YtAO4Ce8?K-Q86J@YiAM?r#Eklv1@qrf zNiC#S!=`e)seJq&iaJP_-QBxjNJA2Y+MUApTjmmK&?~+I6~@5mQs5Dl1LM_Iv3Poa z28%=i?%m5_F~G1`FT8%hpM+vKX=!z3mHwhET)X&7mW<# z&BRSJXP(z)x+TE8F8q9zLz{Fz}*b2-jX5*0F-N~1OG-QaA@E#_etex3SAqu-f_NGlgKU_ z=HqM+PjA<0>M>Mh3YDDLTDq!LogMz_jJx|J$cta}ksgwpqlsWllwidT05c1IhJpFT zFDpAc>*7Oz{f-xH2au3)u+{ddqKagIm(TFMISVTZG6KruRxD}FciJiT{Ai!>k5&LQ zN|(7nU8ws*Y~TH5g;K8Sa> zy0r*hB?wH*;rJ)U;qegh!5SiBPdTum9?-24<$D_NX$2GHuR!|L0BHJ$5V$NsdNToM zt`f;^GFu=7udxw&WhR)K^7oVuXD#)biev)(T~$pqVdvkX*{CtWX~Vp-e{s8*Mf-I? zDD|W6fBoLWoHsoZmWZnq1&Sqd!%9u%&4k(^SqCK$!%_A&NiAfOyKD&Xl4QC&A{+BS z&8m}Mz9BS%H9BTwgOH6Tk?pP-VD*TU?g#vguOu8I_Y96LwI7gQ-u8^jd!qThRvp#O zWktqAS=NJe*z@$p%9ckX)Eo9#AZ@sF5f6>pPv$~gCE5>KSnkQdC1ebS50(4|%S(N! z1FCH90zf=zKRGFz{=b?7?h22H0}9oC%mnJp z5QgHvgbQ>T8}1oWY`TmfEK3+h^nuSr^?X#RhqFb(Lz%~{3XOxy6JCA_l4{6)<)6>6 zXF+X{KPLVHl3TRP7QvFl?%*WYR?_y$OBTF}+ zHKNoMP-__5S#sijgzFx}bv?ngoxLS=P8_?6v+pc@iG$N2d4qg?Lch1=?wmnp&~x{x z5J=;Irs9jr+itTF=0Z!+-QHH38RgPv?tG!t4(1$Yn118G_j~EwuZb^lHT^Dp30Nt0 zv31_$&!HPRAM>I7ePoPBB`EF?6?^)g5P!Qp9;6?B=3K|9>^jA#M@N6}@>TCDpCmFb(#!``FuSzR6v$;&sL- zw%vyd^AOv>SRPWkRYpr_7`h|gkrn)momO*F;vJXCYe@&?F*=f1XPxe$N<=;%RtVMV zfYPc{!ByEWflh#n^PfMy1y2Au-iHf7($!o5WAzt7%`+gN9^&jf>CD9hvg}P|1bzO4 z%0&BOMcYsR0^!X{hTD(Otx{2YM61ppX2=Kt>njbodE z^WgojaDZA7=4|4@3DINOfq%_{AwKkk!xK(ekqF`d{~;mVFQd}}&fncqVIh>;_T2!k zTml3o&O?aklIbL^uh4!0?hi@5z7HB|AA0Q$(+vO!$8D2{HxpdTBmV3VEe>#>GWADv z`vOoUw()xLI*RL(oV+ANVOdd{aP87fZ+X*UU2lq0HaZPtFo(}M8EC~x_KO3W3Ev7x zB>Zd-^#FtEM?h|tUY3FRf!_?o`{#>z4%Z{VF}As<_UX&+{DT05A8eed<0!t+AesMR z$COgMTwCO7!okcs!#)p^jBQh|6>ng;%q|490c}94)(A(G+O(%hXx^YFK3`#vV}-65 ztA%?(-@}tyU&WxOsi$fbcMOk11ID@PmV7GkIV0)5fxl?RaI~U59+#h`3)=Kl>f7tf z{Q{yL-EA~ipp54sP*>x~(`#--txHsF$(p$)Z~Tt0qk<)ILuofL(BtO!Gp|#n;-ZZD zg=Yg7DQ8D~I>(&-o3!$5Xo@pud9sgI?%Whxa}Fzd@j(yG0t?5C3)K84CO^$L%?VD# zCUJmnHOr*NlzWq+U5jG23tMJR?Ne^{oS*FM%Eatml}hy|tI}IJ1%30e3iDXzl6SD^ zlJ%@n_+rJMminOP9Se=8oOfRWwm8pXcCTXVVT%*A++pyNWp8QmKPTS&*0Ig!i{1g; zE7Hr~Ict^fQq304-}u#!mxI_#(h3hr_*WNJ@gppgD}_th!sE@81;1lPAyXp(Xt?6| zx}vayzJ;aS-|mjio}MWY06u}6i`ShPUX}wtK4aG-+>1H`yywo4x*Z(jFY?tJphBO* z;?f}R7ydDhKxHn_rTXqno}_2N(`>msHAkETT<0KSD%HhvhhYWiYE9)2*2#)fptl?% z^c~j=(0g0k0i7Wo1Mc)l9o4U~L#fea0f>DAHBDw~$fj6NhV|+u;cj#kvTG@0Ruu^8 z&D!Q{N7^${H~wX}N>!tez2DRm59*1Bv2CVuMW*NiX+Q3V^}K2G-J(YNS4*KS@o1iPuk0sJ*b2}(22v^ zz@6PdD<#;c##xTTx+y;&Y14o`f7GTTDVNZL<9Uov%6qx_Pnj@f3VP_wC zz0w{5Y9HT6XQMv}PB2n2umFIww}=@)cY;L*#ERh9i^#Vhl`bm&V1y^%fgD|{@TiK@g1JBGf z2^y_(ys(oEt}?@k(IA#d^Ge7S7tLYKob~Hr?R)GgU+#CHS0c^~5KRxI>FkMs-yiYD z62Ej(u{X-H*WgKG3*`(C&lXN6$;LK9R+V!CZ_VAZAKN-hw0g@LXtN)%(!^PLEV!An z@WPLM)h5;wZESPzU+mpL(&xzy&eL+~75FP-qc@rTYLC7X%F;0GA`NcalJFoU0gFdT zO{3-nV!34DW?eOj$5+{?2z1<`KA2$~wt{oT<5GXQjfKK@W|RB4#fy_q;d_PBi^KHg z#WY_T?Px&yts~3!$SCrNKTqEkwyNG=xwzkUd;WUU_?@>iy9wA9=)Syoy^-TJyW43z z*8CYNRDWO-gNHG!@AazAY>3eAXh~AZ+@Oo5i2K3aU&n*tz;9!B#`)MCB{E7c@rb{! zv8&2u6xjEuRYkZ%2c;SIViL4TV(oaDN0O*H)+X`W18oV-z8U;(ZWV1ZD!>2mWwGZ; z0;daz!eSSk^U5N-u(^`^O2=x;y848y65YL?dbO!BsgX*zO@nNv`Kd@W<`PaNpSD^RnE|oywn={ItfN~~BVScJI-JR$d4R;B6iZo|;uMS1xO`jPDCq==mdp?~i z@s(b~H>*taV)7h^AL#+lcEd4A4Uxw|Bls!3%uc4At zpx=vk^W`z)0Ob~}-C!W|#IS#H>0n59iZTR$JdtTt4%G#?=-ey)-i#SDMJua9Gn5vI z)fC+A2VhcXGb|@-d#(4m**omv7g$^RThSV6Ioz;$$Nw;B&3BoTwA>@YI`+thd&I}FBuYGkkgd_S-Cz^Tz}ga0LQ%^@h~BW6D>BYtYiEpFBo z|HKeXLY~Mu3NxakR@rr~I1ivFOIq@?>_#aqwE(5IHDVwgc7iyct z)kd*0GM_6C$<3Exwethmu$(&4xW)556GC>6Vb-~UvVwMuZ{+|_b|S8e?MChnP}RYk zhbJX{)uc^i(ZTAadvlpZVh-gtggJv~3x*$>JVMNYb&E6&sWI2Jc^ah*EWbG(IFna&tRaiFZ@Z2KbPGo8>kc2^a6o%yj^ zSaOBk-FaTD_QI=6!~zDTB!#dL^)KM`gWx`SrrzzNnoiW`_*?}x0UO_K5?}ts&#JdS zaj$Q>4-ZE74~ab{P6|0k4}Ney=P->GpYmhpp9B~d78XHMUva4?C-2{*F}w=R_kKku znkV0L?E-F`FQW*+je%P;9|`&@+$W>kN1v3kcL_PJTo_9)N;QTyJ*xpdGt;l>ozi&6 z2qG3&D}JrK()gvhL??4u-Yw&=ES{pr7IBLF#E-zrGHF=L1XpTd?GC?9|EYhUhm{Z` z*BZ|CbsX%ySjoZ$zQBY%_KA~WoM|vp@?iTn{L}Xsxk)Jag9A({V}D5jz1kl0RfLY= zRxv)4M*QQ-8jM_l6SQzEQ2p~u4vSu3JX~P33xR6)YAW|qVlIHUN@XzW-7@f9mn^h9r#&AKUQt4xM0Pl!EjkS%XSfV5{(sX)Id=36KQ zU13C*5rQ`5oR-;Tv0?VoXXlLYss z+m*48`sLOCJ*aDvu9=H>SoXi4-kXLy18y<717PcqPGky^KLa;hdhvT=fcRRx{QB|o z_wfO&kmq>5_?aDR8UU3r0B!8S_WAy5cO7*AnyjMs0be!J1x`tK}%wXm<_$9EiFg?bF)+5<~@}atrO>aogM(u~w!;%=fr?JgMuhlo!(Km)PA-Z{K z?E0FuRra`W|5V)o1WkVbdH%j}@d3Pq=6F8$rkU71+7$YQ#eC~z*5S3j3{x+Ne;Z-G z8P_DH_{`Z6#mB;AwlXHdF|0gXWV#C|OP?b2u!(;{=63A5G<{0UnU~>FyXyhjiN2vx z(qmBirw5c_4kWd_bMxgjiq|#SvgYhevt<4m=2Eh_JI}2QW2>OO@+kAt2~S7$(w)eR zLjlv@=>n|)`Qh!}|Diz*Km*4*$g7rMGbTMOMp@_g@Q@zP+?La&+CUaQ@kU$d+c$E@%VS*Y)3*h@$#4Ia%i=~QKbqgIa$kmI5bRy~=@pL%R-9|L zUjBWpy2MJX8k-z4nx{7PNjHa$v<^~@Qi>K+e#Rn?%T;6*RlXVADKho}4U790G1nIV zx{f)11HYE3z6)=eyk?{YxhD;!(~ zwXe?Z&>S}Tn*i>amTq$6{7-w?p77vs&fsuJU{sX8pH$P&yN<>-9fThAkR2KoB{wMM zK`nIubE#jY!WgO zd1W$OJ`mZzpoxS@HHk9D^b^bqB%4G_S4 ze0sZ?=4E>V6mTt@`?G_#m_T$vp$;07zmsmP)129fOb3zCzOsRVPeP}TU8j8pwi4|DBJJ2$^PPJfc<{?J;40f!~L_H8+c6r z-AAl`BmBgg94G?jvPFcmF{KIABNwkPeVpFqRo#qSwM0raAwWJJ=`&=@$#tUd@L#?} zoMNn{xzm?a=uKgJ?9|o3YVbnS3)vblN6}B{_<*1(SOTl&Lou{d*CF3APpL$i7W9&( zJVudtXgR)WzH_4s)gcQ*sJF2Ss809Hmem_EiMn%QSwfhe@*0@3ndi@_^0u7WMw!ts zyq;%V5@~$Qse6=Wi+d2a#y`nOe+f=s$^V1;@SBAdVS4RjVC`a&H}cKPf(>DM;p3+@ z$r8U4$EW_wV(%5<7uPv=xQgve+qaqc=jizE{yr%Vz{xZJ=lQ)?9q}VQ@o+B?uor9f zRqI}GBJe_dQ9@r!NVAB?%{@c_$b}jQSBRi>fD_Xoo!5Ew6OZSdD<*XRLX(nrWmY`2 zs-evuC%c0+*EK&gd17`&PlZ>l=bK(mPo|cWUJ9WUuc7jXDt`;c z6~rLK*^;S34~MUVwVmQrWTg23<%iK6gj;?6wWeYyoExFdh%rZ@X&vrwh<{gF*Nq8c zH0dbmP5&k=rUkszdQF4hoj&1%h@z z5cty3LIl3pNLAT({1T4PQ)n3}Ck`@pU5Ml3kxZos(u1Av)?dAF+Lh)l_6_+o!_o12 zZE&WG8=uWD4d!<1TMPPn>-wVgFE57Fki%ebp#3<~PIG z^mqYb4rP^67Z>F>VTm1qkuGuzGVUz=ys#j>%^D7AZdk@v@O^OCIwY+^VqyR1ZvUqn z$dUSZZ5o%d|8&gh8a8W{1IjX=F%u9KvLn?^RlK*&-aM3&l0z#^u~TY<7E1U<>@DYN z={b-%t}58nuPhi|4NiOD(N?Z!uoe@EE-|u`kY0O;HmyE4`HZWIc^+C)!_9u$B77~Y zI`hqrFjQ2c3|u~um~C509g2*A+FGIBU%_S=r~JE`z;f7AtZCzBZ#3o$s9JF21N+z` zEzl;wywHR$|Gble5+}F0AXC7-#c&ro9D>owbQvGrkU7Y&tlfWxQ>??S43aADDZVCVHSj-migI?Pq z8`5v+9IQ9L_aLF?@6yJ`#ei7A`j83|0PQED2K$~3351~nnUmz4o+~*uwe2ntswu)g zuJd5dpjDILjW>@b=+4tp1hw_)YLNt5!_A7D1QBBuDE$d0rkpzu-tY~Dd^_xL#SQ&c zLNxOblXD=s*&%W|ce;w@%yhq7nV}ykeAlG^5cmB!J zEs**LfSy4?LB8?3cXE>D3a~MxvImfvs(%KuM%ZQN^|x*@@Y?o7@SlimzkK?bD<-F= z6L*-6(l?G&tqam09Xem~9p!uGAwT9i`;*1ccM_`UKxvw20Ci^__GAik<<6*}j!cI$ zkyz%?Bbal-F$AKZ0PyvEe@SK7C5UuAK!ZMf0es&dalXWs!b}*15EJlEZ+5Tr?u)`xtH(iGGRdM+V|QBxwk7 zC`0H$gLiY#C-HyqHCPx0uQ3)wx(pH)zi59d;VlH4?EzK~vsqaiEx12W<6sM`5btV%dX+dNjM#zL$XP|%$8@ym}F>s(K%GliHVv_4z zJ*cVK5A?B*m%V>}OY}PV3;FV(fVeScO;)__`FUI%6n~qO2$?Sk6;%Vzd3W8yfe|zl z>On-#wU!!=b~vkWCu3mH`ke`P5Cg(xFuR}brG{UW>F*Cw$q$X(wC-YQJ*#uLW_nJT z3OS=DyZI*XufCLrRv1GmeuwJs#0se_qbaG^uzyZmJx&}hmGk#uUFqBMQhPNRmnlEo zv5$gs4K05#niM@SzxG=-%q1R{vi|60u1lv_AMJ$~KkVKle|QkR2Z7e0r0N#R#KV$g z?WE?^DLsXZi2Pc(>~}oT59y*MzBy*C8eatT_TJHilYio7cXi@eFHOpdMrkoulJ*z~-QO2MoLb|x-;JFc!zlK^2jsLGqj$OW?7R|lNYe^kDLyC@qmZ!D(D*Cq^ zaIOaRfB8F*J|}T**4{O>D1Ofe__HI=Rjbzg4H$|=PG9~`*{ke7O+PQwo$X9~O6mu1 z;X2m|wK~pvP24o0GZm!J!;^Lhj+AmSBY7f&gzdKCJw8$Btn@V0bCyj=tkP3`#+XBa zFqrFqKA7j1)*#HC4VTT0$UT!-$`9q=xn-woYfnx`Lgl6^fY-Qi8ihUC&^9Zj7NGq4 zpn6Y7Y|PNGk2qrQD_1?~z`i=XYPC$aZjWsWZd~0AWJ2x7uU0^^el%$|4Fs7x{?(}P zUO@StuR_MfdJfvbnclqgXBh=@{j!nEN0Z`Q->wegfe8%Dm1=oT>| zdd;I(xjD*NdV;X!qnFk%Lk<=pcn*{~4e}+37>-{AiKt1XX+*_Q0?U}o-5p2|juK{roRSV}-Nk`T@TL`9O->_x=?u=n!MX-Gi zMcYJp1%06{F-XOo!*-FjxeG)GYF187cQSk)<|3b}%3%PP0szy{`fow4vA8Lt1Bq>k zWI1{hW)++IK;HXK$N%SZa}7b>}fB*o{rScgCK-5E98dKm`-Rn4TSefT)^N9v85 zbmNwyedv!LyT!gmF}5~5jCxM-Y|RlfIvBh00_^o??Deu6m~tR|a>oNv;0+5~L7t1s+E+30_90Fzy_j%@q0YVNzQGWX{oWSOc zUzLw9KN%_go_T0J@-5Lw-#|*oAc+UOA;u3s${ZzC@@gM+{qABE3YzWm8bJW^XpE~& zzZhG)%^p;f+%kiFC7gKhq1H_0-vF-JX0@&?T7n1?Fd3L(B!RktsSGZOQCfvyhsGb z>L!sZWRa3(!s>E`2>R{{E^&Njc*ssjcJ7^77VcXomBPeG!L1p>FaCH9Pd$x8RT`ky zk_z|XB1^FwAhd3Sy#w5^0Tl2i<5q;oY;Fy_OEF>k?AlC?Biw$K5Qzcsx3uJ^I_7ut z3rjCO4I>?tO?4oxws0IcLuh1H$16z3^0W}6TtPuyV^v8<_&<)u& zTmeaqCh|z!ZiH@eK@8WCHLU_Uf)CegQ6_Wnr0HPSama}P1f|dj*qRy=@Xi+qPseb( zDJElOBn18QFBFNJFk^0wMTeJgW!wDV6q>l-%!ivK@1dOdMNmyYVu7XY3-^!e!0RTL z8$Uq9%g^@F@OmXY$4C{3n_TYAE*&@;77s7v9Ny3|y~Ng6WA%X!QazAxJ>hWFUs6*z zA!XM_)*NXuHz<@^D@cp%MV=qd$aU}%WZ}NrT+=5Pd5{xQej0q7Z-3(F34%XNPY|%o zQS||D6Hs{G^CaG52bpbE>E2*d2hI;F@3R7iQF>3}X8$!0B@n3IT1?2iaA}GlY^1+z z`sl_8WCaf6seh#%2$PdzJ|(EYHupE20O)&-Hfl2VF%7Fk_13uOnA==$y<}>wWQkoj zK`Gr%&+^>Zw3sm2WD`EE*s>v)>SIj2-c0x9_GFr(ej-G2fQ{}!H9@3qngzs+4Z>6R z%v=2(2TfAdQIX}Fh`mFKRSBeiXVV*mXcj2)`yO0gdP7H?5TP`WRa6W0$DtT05z%l0 z8Zv8D84HNMK7JGb{OJ8R7a?kJ|Y4f@Lb)jSrR>_G08tx=qCCk8hP1?;A zdtPtT3Ze;mNTO?h!fDrKOG2VbIvQYZ)w7e7MxEj^i=zUi z(Pm0zeMs7g)%WUpo__VkdQ zdurekM&q}t#;>iHiZOGD{7-YPHX!9o=b~F}-<__ep5G^B93>~MItJ?(U}eF5obpOK zDCTmrJM1ImBvJi&?)rNlV`HgMj+tnhDw5&in}cGy#r||)9U)?D*X+>lG+#s)0s2da z$fuq0EG8sCRi5Z3b?;s)MjjkhTOE?Rl)tfIM$DQldHHpDQ#Nnc-JlKx-dgk!B3MFL zAsjG+c4m{`aaeX)E@NrfvF#Dr5+?UJ)0w=%7%kq=i3y{(lLT9&r&7>_#|;Y8dXX*h z5d6K>;yK9!MdkYtftAX*lh2JYFIj#@r<&q|^17MH?jpHih1tg=NTP)cnOp5@w7VKsvu9nlcf@T!%N)B)@&xoAfy z)O}QM?1;5DT*7(=vp}VmTI|@Fe^o6lxlJwqN5`50n?68{oyN*oji08N(Mj!S1%d^& zlLDP?d$|jPapFt(uM!Igt!762KqV0B>V!u&*wf|V>>wC<#_lrKZ{z@b?ZsJ)RoD(Q z-x)(n)dW7yT&8a(GgINg0Ev^hX&En|Tn8#h2YRxWT6fJArQ~E(RNgp6Rv%U}A_7wc zWQtdm$yp98KIYm8QygVQvE3)^f0V;YxSqD2Jw~x2C&sT+3?)zn@dD%>3Hb zWTBSRKg86tri0vF^#WOPFv4o7toyHzo6f8WO|Pmby7e4&1YB&BJ`@DNow^IHszLW8WO1C_wn2UR^H9nJ;D>Gz~EOuDOt z0=)4bPw~ntEJQ*(>nAs-Au(U>mnf|hzyIv7t5K9k>5`K~s(Tu9^O%zipZ93&mW|DX zD03EljJ!~9U9<0pRv3h?F@tW4*hW2BkBRIjDDWa@oz9aak*vc;zI>Aoz9d9fr+zkr zf&{bjj!~>sAMDOHQS^2cl0W7}y#2oT`ow?q`dU1Vy8Nx(?ZJ& zNbB-zv?*?!#bgY)!&zp8GYt}1|xAc&5+@oY`W*S(M4N(C< zkOEld3bSY@5l144`XlsOIiHj=IyUJQsXT`HOGet#%;yswInq&GY=ym2b>*Qc^@biA zReqI%Xt!W?=Am`*Dc<_}!)XaqtB^k8_T?Q!Se=e-eSWEb%@u~p*3?|0NTc5Jt)aX9 zCrH(FobcR@nV1!5O$o;eXRR`~ijsgnYdFX-QFh<7yQ-&a9_{~g&@^f)!1tf9b#R0} z@X-<@{oo~PoB+=eLf;?dZuU&9mmy5wX0WpgUg230XS-jKsv6Eg8q4`R;|2u;fgE37I=lC?=Rg-`CM z&9>c2M~P=fzRk>m;1-&mi$20RI+w7QLS`ig$9kA{Ifspc^?f#|N~T{?l446*OVxxm z*`Gn*MNM1~G6V~nzD#8^ig@5;v!*-+l(GHTDwPvash+HQBY}Vy;o+g^w}29OUg>M5 z!J%n92|Ce2#_EcqoByK&q6oJ$_306+Av3kHFjrryu;Guy&=*v@nl-5>oU*{2c`; zM891Ivq7JLBk2k@eH}fxvhjJb9WEbIBaj_kI3keN60o_jewNq=AS&Y5LuqxSb;@YHi`4QOnaz zcS-?C>KGLqZv^^xi=C%Q1|5b}6!_-vN~(3I+TWxt88<*<2@TU1{A^j)oYlcU!b5$6 zAJsHlp0!{##J?N*8P**1-81=)e{6X*;EIDG;Z-XfH^*yxbd+VlzfFMN1)yc4$w9h$ z{%GJ9VO|yEuS6JS2|`f*otHybCd~VLasg_3c#n|*ry{HAbwY|?8a|HVM&1TfN|)EPW%qVBl{;A-HVim z!>$zn$r1J^s7Ds!6DDgK91A>3H&Y**%6_|>41o>4MwclD`6CEfBzRLRVxs_gXUK}v z_|BnL=TnDiatZl64Ooek?lul05B7&{R4PZf84bmx5D`4Foi`wV+}I77>i*_?t$B<+ z+$6Rui%<~_6R+$Rt~`8xu|e3WU_?8V&eH~3Iv!w-G1L*PQzXN9K-VT+=_Al~t@e%2&0@Q0~# z|9DG66z>vd?%V(S9_hf9Pf<*)*>*e4_M1~^*o#G2^TE;_ozLAG8M4zvFepN=UNr7Q zVlgW*=|+c$(V5B2oi7k0rp}BhjOCMW7ZW#NqJuihud~3u>B#JbqS%Wrgr>cmToRgY z9d3=IXv3|@{jTxvtDj1U;YI=e>44KQp`h!hAK>%SIX?EM00cMl`x79ZvwjD7<64FU zeiZVz!v*s|aoVqqL_)z6pq^=sr$P4g*Unqnerp7KD=tMq!=1_x{5)iE>LBcdJO5yY zjk<1q(j~S?VatD5nz=ccTB(a{DCrk6LV~Z)!YN4SmA`e>-V=0C}f;xqC8?OP8uitIWDo6qdQOr~274sBu@w)V{m?OKBB*Y>3hHq{fW^mq89^nWbKmTRrdW3nvy z7z!@BXXKAdo~F|O(bgKKWVlsS+hw@?b!xR8pc3kJZlu#DqnlGEvv*%I)UgJ?3QuSW zb!B-tXhAy<<)di?*bYTC`b{?gW)vVx*FFs}Qhz+|zBh(gQok%nYB;`{0@`5LdJFEx z&3U^!a*T#3ud*VHw+Q__7ogUCTo%N2H~vDJ$*hMn*h}_Mp#Lv_$*04tW!&5P>P9X2 z{O21JuRceSvo| zVRdsEk*t{=vIhL#VpY>p_Nkg{=}^;>k}D~L0VO<=T!W*^ET5_698&%V3KDnj<2UJY zCszW6kqzvvC=uPDyLhNcNs&8artp#ZkkD?x98acT6fmTGKO%5s8kbNC!>i%a{^o4G zcz~!vR}$A|7UKC28vf;!rc)uEwUdA5Dw8A)=Y)`WAj(ty3p*x$yjJIC+)mKvLdL@e z5x-LzO`t<$F_xEW9wN&9=w?~sXPyR)DvjwEBfNc)$Cb-~Z2e#UO!m9#?X>v%n4IW%}TloKreLCBPk2A+T#GZN1!-hs}v&T59MiH|_$sFv}vWO5PE6(W$-m{$p5 z-;nFaP))YD)8a|X)%DYME1Wy6+b3Nh&?mk6P~gL;Y|~C6bjEy0xhahfJ2R3QXs0}$ zC*$Ms!&^kw5~;CgQg?(R+3dDZQT)4P*CFAnf3ArzMtsJ_3_uwXHs_&v%aCGvZ3Ynf zRYbicKRIqq1UG&?w-ftcMy!Z?TTFKBI}$MhitR0IaVK?wSO`4C7;-lW(_>2Ch+3#< zfnj&s8j4C8aF#>4Q-2eS2T-x-*MFibboIpA27N2zlbB0Xlc~j0`U?YX<>!*Ik-_Z5 zuR@i?AGMKC$&a9c=x#_R0&8;*!MC+`QYVqt!={B&pq8B9d*J@2V8YF&DZQ#!mq)Xk z7WcBo90_`~&xJLs*RfDo#aL7}J||EkaKU2tqu*!|mzG!tHWQ5*FI~QGK7@^gwFEz7 zmlLT!I9pLOVPt-qY;@ibfhWX2N5l_Wd!%c>Snw6Bai-P+v9s7=McPhzV9%5Khpd3>f^RX33cx*eAU#vaZu%wnn(tD5I&Pi6 z==a$A0N2*^|9I)u8KSYVX38>=VXOW^Qw?{WBs$9NG9Z+w;mStPij=nTSZh;J4_7BA zA9!&_G_FiX0&$Z-3f2_b396Y1O^9Cxvjbrh?Ppa--DY^WH=~djLEnR{kA@V!L7eWF zINwfPcz|NxPhW(>Lj*&>#y?EFSa!O#@GyyPN+!M_zgGdlX-<}52bctGu#yT66u{y& z4fqCX>=yJ+Sw%8qRgBDGkVsakh#aj&ILv>71Tt(l?(j6=+21(|M*Mc8_o1 zYB2n9zo4y0@3ukjKG#k-5_J&6(|)v02^ zXfOzh5p-X>h4cRloUxVdM1gN%nmm`4(P{ zx!nb>G*J4LoWifa`S9&Gssd*gWN*2Hs%lM~ppR1@e)Ka9FMzC^(+@xPd)}pK0Pb}C z3{W+TU@aSH6K@f|RFdEay-L-kZNQ3fE_!0~Lwi;sT5%&6fh_xpt5DIyqw1g{RS1t!N!S)M`B|+=S|i_UoUx&t0n_b zT!GSt?!u#V@r(2Hi-*x!&x_JYjT*c4gd~#hT8@iW?S5Rd&_~j6famcLIVivrg%VVe zWA_8tLit+`rsfR*)kP&o4YMH5w5iQKzqKQzb+Ugn+|`#F5-+_koeThvc@;pj7lZbd z8%`H>FFbMMekQh5B8AB4^yO@hgU>&eyUvFpQ*s&0t_@33Ss;*Fhix&U5b7>IKfa+{ zQ%vAJr|T8QdbM{D=*Lr{$OIdh1M40dQdPr!X$Dc{BDE0RlwT4lDS7TxMY5ZuPN>CX zh2eLM>$wHJ-bNePS&o)87L<>5*5yk~5g|1I`|KCb@*!gz=>gnQs*glCp8EO)`LXze zL&|BHS$3=N2oLc0ZeOZLaxw)KCk~Myh^;T+t)=%mj9b7&=1c(}_EAOoBap_2pTau^ z#)%f-M$BLoV0%b)2SA^?v-j{98})^IFaGNJID3(sxq~!Cw!SjtRp^JAzuw_<{W-^V z0l3O`^I(sesm(Y*`w7TbT3Y+;I(#m@P< zV&^x>(pZ-~1@|<^JeiA^(kl-25F3nU4iK)DbFb2v)<7LV+i@zi$>!w8Ja@IgwNkKl zzvvTre`=nGjk9UIf>Q!4v3Ypd4s2QqO~)>_Cgd2~$)}_3#{(lSM)DqOVA?^%>XDgT z@<~)^ic4y$ESZ-d$$!lfg%V}E=Q>fyuK|5MUv zvav*j&Hc+gn^;@|?EXAm4IONtw{~J?4tZQc%hu=bXj}qHs+ou96ZUqTRXQpQM4G?l ztZOm}z_^&mJU4pXRc{uGX0IimBd+0#*@-dLx-tDXkt6p7(dVdxD_IWm(`KdoO#_7b3FhrkZc=9a z`$(YBGTKx}=&2`_HThqC#*n2k1n&=Pa8H=segk48ouD*lk{<(kw)$T)>^N2$V()pD zoErxBbt+LwaKdhn$C@yYd27kFM&4##EFtYz75$s z9$bWx35Jko6F0MOZ`$WPFogvJ{rk^uck7e1&9@O^n<*@TT*bjqy|NNPuKboU07|1) zK+o^bVm0xWiFJ6nua(7I-w8SLDy}SR6(UFj=2eX7k7U{_)|GFMAJ=$(GnOBX#I84G2m@DU^B!<0yD=WUa<#98U2eqf6=lXHzTlVhE= zMLlC3^c)|c?yJ2znN43Uq#ks+Nf=&%eiU3+7-AO)>6-~)it2m_D`)E(JHLZt|`yEzdEFw5Z+u* zfXFf=vM?#utW|MB#RPi=+}$~ zG^ivdmXZ^RWi29Ge8+{(2Kq%Wkc2+u-l z321XE>D~0Eq$g!5Ln?(qa`>lEq7CYlz8`?knCe))??b1Jpxe|_AMwZCD-3^|IM#I4 zp6*WuO-ab$PzKLVnppX0Ney(U$}VE$KXT{Z6#)yr+rmlj$i9y_j(xktHWJ=v zss?{4N-^@*Qen!;ssogU3Q8B1xxVYUpycDiU~e_kNO!?k<*yj~SiDEe`1x=m{B3M? zTJ>U=LfWZGVbmRhX^q0JbC53Tbys=4%?{DrUdXNH-|rvVI!9bsh@D;;t}tEUh%u3n zagjIBz1bbd!N?daWH}eaCM{*+qk0k&{g|{|WA{1-NWt(15qbgm8WD~66r5hQD9ivIc zVkR)v9T4gb!Qyc-g<+V!KvW@%fpS+<{7(<>ltFriA-04=3?L~C-48=z#^o}FQv;W= zQl$wBNEc+fpR)Qorb^y%X%(?=g-Go6c~K~g193ns$0R^L^0<_s+*fDTjFG~&T2`^L zf{D0wE6#n4QQ@?=zqwMn7Spl*Kf2y2Iic|9#HB_wL)yZR=sZtf$#Vn`@5W9KCmp&tsFQ>OroZ zxMzc83xSgkg1JZNmVBp${Jw-G#V8bkn{e^dlo{k(ihmX<=emzNdN-E~M#YR)58YmP zrHQu5)P+~_gWeNN(rzCkRtWOD6bj3CAu7p)%#f%F7@6%V@Y}P$U^ma~h(TdKA$c=a z1syV}%a5NlkaQ$=Q6|;K6@9@+9;w9{+7_LA-cpV(W<*B`+R#6eQHFrQ<!t+|3Ks-+TvoHcRFDn8R>w(0D-cSTPW6lD`5Gib&EN<%VT4k8; zfW;?%B|vomb5(ms!BCwEFhDzgVM+nY>KK290n<*)9z~`S=^QL#Lg97$W4sAS-K@HX zX}>!Q>Q1uJGC=F70_C_qvaOatn=i_{gM@SGCyrFGXt19bveVCOEExIE`b2}8p~}hO zp}ou;jLVoNfbv8n_lcqughrX@sNp3QNARM+7&`mkd!+S%R6{t*rA&7%CWXUynSq@^ zWV#0^5@XX-GYK=?kD>2T4Ia8=19Bh|YC#dy|2QihDa9w*<^>FI6^9gYnpjVwb#L*; zh}wIYKz~u&RPyMNQPd(#MXlPYlGLQ&BHH4v6^qX{!~$8v$u+gqhBj6V>3x%x<@c;h zMU{-AA7M|l7|(IYV*bh*_0k^cc!X%CDjbgG??P`gm1}?5e?(=kk{G5L2;!HxM9TVN zq^YUHo>SXucEZ79Yc@4dD#nE6iiYL3Wo-An+vx+1T2r91r@`Yg!0&E2@6zLaAma86 z{L(*{v63{bPuXj!v`})PKG}G3o;7bBOjXAYhbP_$)Q4S$-E2VE=fNn{cZMF@p@;O$ zp(8I4=r%e9mSP$RfG zCKkxq=j`y$f~-K~rbP)GbkB|e^i{flg=bh?rCPAoY8;^_;duCDSQPBBX-ccA4z43@f3k=SC1Ky8a{*hC@!tL{+mKpJADU ztZ+~l2;g{m>-?zq3tPmad3{i`Ea>wWG4TDc?jwZL`yFJtO2Tl$@^_3`GR{IYZmmHw zGWZ4J^(rH3{S3(~n3Y{ffovVwl>j%Qj z8@7ePlj4j5!@EJ?7X?(m1^z{p~NS@6PtO@JuQ-2Z06`m6BC zvcF}bIW~qN@Hm>G_)#!x+UpyqyTclpY?#GxFIEylxce%)B=3m+VGm=VKRDpI+s#K* z*p&Pwom@c~3J%V^)b z;8DT!cF2bgUmL|IU3n++j|fWl0uIl=7raa@KW_3DV*H{is1jZUlSJVD7tu=+Hx;ww zI-p)twE zJ-Qtros|3`cfP?9C13MeYvhexIrks1Xl)@xPunv_Vc30;-uxC6b1zLvf$H>uS>(T=vT?J z<33>Ex8}6xe`1&0xK{URKkx1xU477*0CV(nbFJPT&=o^J%f7W1)q?-S_~KWfw>4x^ zdtT$lFC^|aOs6RM=l}Q+lxz&1Pdb#i_gt1cOT$~O+RBQzwbK1tYgALsZl+DWPv_mA zh;)YzQ+0fNgeb>Q1!fc+brZ%@fh-4-mzon5nb-{?d6tWTPC{slKWke~m1Z zL7Rl%-Eh=WW>IEnU|MOjY_hqj^csAG+xo5xLKzhD4taA3h(X#aVKQ5XiOJ5is4Esq$^Xp_p|L&QV6pns!T^ z#ywrgyVULn8tfg^?7);i1M{8~p~}AH_pc-0N*ag%z!~gsIAaSjT;dOJz5EB{y#qEO z@7z+B*tFb`){y$2x0WVtB|p-v)G~k9L2*}4kIg0m+pIuxxn`7^JBTS3&`88F!oCLt0W}gY?+p&V%bEr`%W0 zcq0E?iv?p*P{&)bix?x$|0ccZD45xzVCCcEFS%E-PYPA9$}{PI2b4S={}bMQE!u_} zGi;OA1C0d`uJ33o*Bhl?SsT-o;yhj%Hybq&+m#y>W@XS5D9x~r^bvDXMjv_?wnM$- zfV@C6lfH-EB^7&;^m^gD;U1c;Y9azoc_Z(+Rnp6x7-AI1>pnqV*@SVA*(akFwiWc6 z>}4t|7jO?bqwXC5?TCPW#HfU>gDdTrZI*2q@a%B{nNENu17w2xV(X3_xxyJ@c$kyio__=qU$;= zM#MV8!%xf#NPDr@x4ooKEo&oG5-->O%OxKxjVtHAfDl{Gis4DyFdgTfKFO|C5UR~& zA=CD$1PKtc@)f!SSCLN1dZwHW@_|hyHi%wBEiMR&sn(l4G*&_uYwwhKyODD zE<9cHTmOqZWS6gac|OG^fd7@w26E1g$?l?WNYAI_$1)9?@30_|LM3v}zR4}C4nfMeYFqOz%D_tt-DTiaPoBCP!de_rOHT;#n0+ z8s_?WWW13%^XxZj|{@SPWj2aqw$t9DUeILY zIC;z4q~C^&*1gQk^~!_jZtgdRI2H7YJCK4w_{Cg ziYni{ZF>@zTO$_#7Iuu(R;J>)PXI6vy^GoOmF%$jkT0dqkE^^AMO8PV8=)Of0rEj^ z4w5i`F$cvOh-54=Fy3~(Y$DMJY`mpdOV=kEDWbyc_xBLkwQAypnYJEN4T&dLQtKLyx!gKfM>fntQ7}UFzePr;F z1!fQPu5gkLruC5lEkJNqlJyDd)v(k7L?(CA(0XAhg{7sUgM~j3J>e)o*N*Al#5ADx z?8EAy=%*HH<=IgPD)a!@4V?yUh-a*>XZ+8?aWJF;DQd)EOFmZ83D|%D&vA?(PIiP6 z840x3CQ74)GcBJ-GiqDD+HJ0Hdrqsx`!a09x!W;o@C+xKva~fT4(C;JWziQS;g|mm z#;>L(1`tgQ_UkFfhFzMs`!TB)2<)@T966Oyk87*VO zA;7b$Vu-9Hb>-~N2F*r|>|R~ZS`V_R5n`$Nb^E>#LE}jG`vqNmyyC-TGS&n(;Pg|) zaKPU`D(C6aln(FXEz#3r_DU(n$#m#C3e9|(k`0t%!?Q$^*tYPowz?@la(E@= zsz-mFc|7}gBEUu2c-1si@$fSS62eMKfKJNq z(wl>WM3Sf>_(ymgCpH;2#%jXW52;qFd%i$V$azkSL#j858Bh$NHD*=W{=p*D!oo{{ zI8>Mbw8l6Z)*gv!HQ`B-hDX}HjFCVgNB+-zl4`1(>zM9ZP${S`SI}xaWqt{@SsD<( z)&Z@0h(CyOBM?@GJamR{)2Ds0ennV?(Il$Mo`~|$;PMSj&IKY;0JrReTpT-{D%l%nPN4O{P9XCWu9=wJxkA z{(K}St#VGyZ;&Ls2TQ1katPbT|G^6mleBiDc#6yyDodGxTeiNqprRLd6DDQ@$6e zMwCq{ynYD*taPnEr1gB0reOixMLie%g3@7cVk-w714FBng_*w2z~$R)^bb1dwbM^1 z0bwv@5S-(bTz}2A(nj|%aH72Wzg#YmGCo6S*h^E9l)yMI{1s>c_`!1wA0)IZf2ea3 zovKcZDI(e0eq0^ZZL9b(!723^I&Q?uJRybO{Vrs?`nK@2$_kev({rH`g`dgTGyVh?9F(lu3el8EZ?cDn%3W*>8q^2B&SuP}4*XPCAyxjJAT=7X8vYZL za=ITSbIw8vEa8$@RLrbi|Blv6%}D}q8e+f{(bx)^S4}b&TCJxcFwl)wmXu;4Z9FU- z_|AOR1IV>(TuatIw^&W=())J+VU%rc)7w z+k^zwloEhkSo=3$hNH&Rnes){g{_@P{F7KL>`l5g4yu-c!+|M7Hy1`IgCjol(uWIO*gY}Wk4ovi)7TQ*# z#`1_^z~Gfo)5_#RocYPEgtVVQip25MsIFkj=OS|>#%q%?1Lg@;9t|9LgECeNT5b_q zrAemrai-@0+%dMe)I+U=VWS%2(v5ONqs%=CSbfh+TFIeC%QXB*c@b@|mK*SdW2yxv zH!Y^pt?4$^RRm*?lU)i~j1`bp4G`K+)TT+%h;4P_9e$aUo{Ua&2!pkraE{iW!bGFX zto=T-P0Woejb9;Vy>eme@5@pL+zydNquF#S<3AhN%lBB;g5v z?0m$GB?F&ObG}6ObSYm{A=zixhLluJhrjJQk^!p{F%?bfl%EbD-!55LSD=Hk_mKKV zGeIij1SZ%M53-39Y+TStZoa*7WcTQNx#MZq83Qs!VAMe;5M3tg2G9b=WA4U0KMGko z@}+eb%UEz2#Qk3*%)V_CiatBHCT=2sxjvC+qPd-KJWF``)=P=&-c?y=hvn50~*$Ha2L${g*|J9 z(77Z~wUlqei9{IT(Q$4EGs3gT0xb$AdC^tS!!2Bo_K(1Uwnt62kafX~lusOn{mN%p zOFycc{AL@9C8mOr!_j6d)h1XEhvhL&*Z)!Wlt8*rZ2Df{LubR})kU6~U}5D7{@=o< zegs@~aC+DfP^8@H#>Oj{4T}t=f#}z)%M52LUKWdZjZJ&;Fw<7o6B5;!>J$a%m@d$& zqq4V5mH=CfDRE8*KB*!IgtEP}b66$n;4fxTBTs%p)y(3^_QXyMs{EX?vi!W#Yt~$L z;UGd>+et3xL7uXL$lo&{svPL>y;>9Y z89SLp7ML=pZtb<;gIBg-ggzGN^sZmPtDblFO|B#+FaMq|+B@p?KkchY)ggSWR?!e}koijLRBD(3 zd$;S4HxR$251!N9a0;+h-f9H%V9V)81g)Zo0vH02lUo`!t)wc%ETLHs3RelPyviS* z7W*f=9M0$!Y{1U4O^{h5m$-d05OgZNJJl@i*IUpHV-_ic%?Kll+ z>)6;>MOj6#+x9cxuPq0rJDhbGEO6>ML#d70kTZ+|izGTRD$Pr}BN)*U2lc5LH`D>9 zniGq%11PWNswxKZ4qdY-p>A@sWMIQ?e-lVFya9h<1HZU7mX-y?D zLizkYloNi*TJj26`rAvz&TQV-RO_aj2cqdi^p*TgV@RJflZVnyrz`QXmv+{a0+_9z zLu|_+P%oPA2_Y|}+X~QbTSq1b#PIjTaM_&SPAu$#xWL0QO33fkvu#Ri2goSP*+6=A zE(Mvix-^aWgamFdCKJo0_B^`GyZO@}QK0crG(x&DFk4a|O{heW&#w4x{GgPGm#YWw z`Gwu|{wpfq=H`-)cx2$}-{P>w{t;=*5dkNaqLeYV zCOWE_H~?$Cd`#KRw$epN=Puau(>YC;8FjU6+nS8lJ&Aroz$i)YAC9ta(;aGCEK~_b zr0u7+P$N4uM_x_x-#G#m^P4{k_&wRjw!Q`r|{!hSr=+8~r{#mFil>10cIED{LK% zc>n}INP`11MgWNj^AMpXUV!`O`LV#=;RiF%4}Y#PyBAye*cc+KFwwWbLMPhB>`o`2q*VVAJ)r>H^2=nHh?!xpQttH+zFe7;kh- z`kSd8544MNR1Vmuadb1W+&-HwTMJabP!t)1> zAC0Sj*^$If3wuk^t&i4(lcVYI!ym2T$3$<19_yQ{t|n?5&BW;IG;75dcJWF67FbTa zOeEdc!oyQ$_?IOO!k)1B;@E_aBpaBehKO?~ePO|NU}7o$iQznD)K$?MR+Y5K3-+7n zn5K8xWtShZdh32v?s&Yl+xE`GKgR0#hb3V4>5t<~bs(PGyFTj--2K{nIpW#}@^TU8 zwtn^f@t{*w6`h^0F2m=qQJl}?1a#3(-GX*d98X zBWVhF89zPkr}?$pw1kkG70sZUJnwpBTP_5?9NzAKN+SlK{E zHfvY6?8Qui5u|JRj4c_S{~aU%T}u!-NP+g zWbBVgtJc5r?g8DUcGtdf4Kjzhtrl?f$2L{?-yr$6N>vxwUTqtEq(0Kg7?bb_b$}=vtU#0@c-3#;7-w$#pv& z@BMs;i2ll%5=)vzM>UmYc2e;~Vg#|ScuUiG5f&L_1g{}K{ghl`Xs7{9@@%eQ-gMzq z*LTbMH)veFV;9aqk1r8b+y`EN5!7hu6H62dHpdkOFeg3@;t($4SoIyIWiVy=tWd0< zp!wbS@+VQ!Y&N_F+PMc>BH_Vi}HAT!Tg2u466QqOE zDUHTSkf_)V_3Q7r2~E85K8k!>FD7wD0Xn?77xRK*v_6HJo@I2_JoU%Ewk+wIG+>oK z-xYH5Yu`-V<^wR6Z~DxigxppbnXRxr%VvvHEfcioU9~9!3AjQWr|M z@&gUB*EMTQC05}enhWi!N?AB}Z@4dBP(91zn`NV_%E0hA(Gx#$!M&fr-Nod^kS~8WE`rv# z|E%6yA4LSbJPMD6ZQxFw?5Zm8Hy#k>KDFjg5In!Rth@Uk`}WYWZ1W9G_vy)?lrGJH z-wNi19@)79N!~};_;O*_bfAgzPmz)mbC>%#+4)r+r__=T>+45dEbpVH1niI52ijkr zUpi)H;$L;q>=QufVBtyNKTF4zT%g^Nb}G=~wLn95+7})Ck-qDtb~N6E+2R!)wFCCk zcs-4!XbM2boJ4_87#TW?4+s7j9dmk(QDoA>F|`pkMeJVMHw^Z!f!ztBbVM?~6eD=6 zS7pH8;oF>D?3!XRb>2C2j_+#Q*da|y2NL??cGz(kjXt(X)=D%6{c0~%?;wZv_qqKz zn>G3z@*RI3`Ybc*#9bQrVEPn7q+!LD!77gWe4(k0{i3zp3F8<~ZBR1DXQ-b&R>(#R z5@iW=QNNOaQ4uRi0-!$+&QR#5k(k6SsfiJZYtTn{ zQH9524LU)zD%HqyrMSjj&VfMt2@cjIbX;M==)im?>VXI^G_8qS{AS)$P4NzmnVA;!a5fsf9*k$

9*42L{t}2 zaT*qoDGiCxWKeB>vlBon_lgp>$Or=EwSk=UpWa{%X z{3=*2$(q6q6(Q8E{mXGlHW?DYNP2$uJ>7Rp*?y1BAXSIl8TIun6gX-w^R zak6)S5}Fhj;aY=I5|FZ!X)aP_oJa*~$Zncuvu>tm_J+faoXn!&iK2Ir+BGmth(X5gzL=$Ch^*Tzdg{ z*uMMrUB(sHKvb)w`{My;Ul9L+m|LSbb2;+$$(9!eNcs%ritP}s^Gsei`mjO0P>-K^ zF1rUjw6++zl1xdS(N)xSvo^emeDUq7G@rWO3m zpQ|EtK7Rre&o_xo$r|DeGdZo(l6QZrRi~`v3FFL%^$#-ZG2^Xj>06{hM$$tQ*Eb*4 zrs{9_;RXMo)sINCA|t*Tn{q=VGL|Q&O|_?A9sW}xtV7SBd>pkgHHQIk4V5D&t_r2B zMORA!f)qa8Wu@)%`sOM`2L4`3=A)*r>1~md0~4P{y{GI(Y zaXOXKR+@CTE4FVL4f=m~qnL6!~?kmatEAj?*su(!*_%j_qOi@*!9 zY|1;mra&cat0+6(0qRY`IZ?B8J7Qf>ob9n2-;Rn*lh+HzRBopb1r-9u5m{yi$PqX> z2;i`dSLW>kqaCWf&+^u{WyRAko~*|*C}=2#+<(7;T@i0VL=das*5al<@}jtPF?Ry0|u)0Q!28qy~tk-C?AUY&kCR~O?k+>IyB zr+{S>fzw(nmO{c3y@Rezl2dxpM#{g!$% z8X)q zlomOr3(p{o#M1Dw$EkUKin@oHh3(#u;>yEzSY*--_+VBSIDR6dKLd-3_NBYNI00$~ zKy!1=J#Z4I>QUs}@hpxWPiLF1sm0+pcjdjr5ro&Rx5Y+e1_R*Gm$aY}kQWyz4(j^t z`^dHQ^{*;%^VO}tUP=lZIA=QFWJxlHjsgW5jSt)jIlkon4Aqese4nM8t?BwK`2i(G zn6!MFpPB2OX^(bnUaf09C{tf~N=*md>xYzdaR=Y@16S59PEfrAOZ0d96_PlzR>}vh zdG=f=AFCd!53b?SX5OKq4iej;NPI?y4dY|o_!#Z%tcz7`qdC41tEPb`w9?{&XQ{An z3+tEJOHHU~x3%FqKh@Ed*b#~!l+gLotEZ1qKa7^ zgrWlmDn+@HWN%3BVG2MmV9nrKK)Wj3d>Sw2O*1g4n=9F(V3BN!B>W5_wEfra5zcq- zH1cK2k+7bF$vf0D%pU6I&!aa``}B|D->_o)j#aj=UW zSf(6OysLSK{Qe2wXR4nmvrY0p6TYv@(K<2J&B3sTkgsRh{Nr9pYjBq1M%F6$pbGvr zKC1GwM;G7n8iABXb!mwI(N$;~(zt5gE!WKoHj;-bsJK^m5Q$U<KcQi&(I&Szz znV>3;olj~}T=camt+=^B+?y6OBS!PLZM`@Jq_^Pe` z(2ramz&;$+oNd&VO(I}`XNGWJ$~l64P-CA!Ka96Sr5VAg7fh{Gd?1R)oFCf%lXk}N zheJGxg~IanAinR3d^7HaBB#@uG}%gvwkPq9PuNa!N8ZYH$_d4i5rw=itOn^~rU5k} zmD<;^#lVCAQ_l!D%C9{Bt`3AnO>^1{aeR6^WDiAM7o@mwCp%U37bR8$Nvk*#}XEBCxWOO<`BSsmU0#rVX>a8#Tu8 z>ZcF)zZ0wvM3cct%BT@!|LSziRbZ{>o2ek;I}qZAir|OV9*~nu<^;%dk%#aLm>0`e zl~>U_5Hkbb5agHLh5l`h=^l6#7>*21E%``M3eH11yq^!QFL0|j2ohHx&Mi^x?)0Rf zY1Y#x0>xwWIaK9hljGX}lM1dpda8Fvrjrhxz^&`&JcALN-B32p@b4?zjU#Bx0+| z0qdI!^Gbu+j|=zagzJ9QW-eH`t$nQ4V$90M?886*a186294tiKAvO6 zsyFR5dG9YzWKD|Z{E+^4dD)|+0h$N~l0e+?V$D2uuuOxTdqw#ljYB!h$MwHt7R9H& z@QM8)CD}gN*Nb3&U7SJ-OhMe9V!Q-a=3iMZenLCdW2NE75HrF;%GmEQqO-gx{R z_EaIhh>KiI;$aj^Kk!{TDRVG=U|AXkNBN=9NTwt86ns%X`7BuQ(3fW)W{~7-uVa2W zXmE>8-6~~HyJa_Mjf2oW7{huS3uI5(9-SM;u*di{@{96WemA-XX)2Kxw5_TwK8oyJ z3+q=lCpFscQVQmT}Oo8WlMuTxcU70s+cR z@YN4XMg-4fm?_q~;lLbf$>%_y8DTqLDY_iSAz)=+<`>j&+0MYr@DuYif+!Bc{AI|1 zW;hz(LGGc$!yqZ5LaH;hY({ugMQdw9)yTLDX4pB^Tw=-QHl zN_|+>P@uwF@0*sC!oK*783L4+U=$k=p_HtNucRdutb-S(n_>w?37rZKH4>_m4XA_s@s{^5EuX-?V=KORofLK4w#Zf|Zzy zm7t0Wh{S}Q5+;EqM*n0_+-7w%Evnow`TU$Ilcpv8iNkF|$rhJk11NC8s&ofdZ8)cP zZx^q=3_eny327357GsnZpFk^b;E;K489i{7^hh#ELp1Pf*R+}UhJze4Fl^YD)Q=@o zn>`6tyX&0LB#t`mb{Vlpm@#885Xr+6ieXrhNg`pE8Vunb3G828#@YqaKajnxMXND} zQ5M&j_;rkq)BHSyJS{(V;onEylLgw#NALMm%JbHo40OMRbl}%Rv`twUo@-dMZKnSj ztVxuZzC{i!uLCMl$p!H*rQkc*cTAPHSi&OYLIL34uDB$VeZ%QL!bOGbtRE4BjA%yArLIeX@Z)v@KCC645-UKTU^1Mr{T2#I;r)23)b+2&!k>4L? z_3nFYzyufD&;wJOV@+|pKZflqJhdlS%iU_b5WpH(7ms7})JxAv%BFGK7>Ns89Jcmh^we*zb=SqkdK zg1V=7Cr%!SrRA>+L2-3mL#M3sq~&me{e&?ITzZ|oU!U0p&QAh0z69A{fz$Z1{GgC~ z-qVmxZuI%`;7!ux`P#tv07x8b(ZCNuTtXtsTYr4QyzBOl-Ndr>6mUi5p!CV0yM+}& zmcm&0MzX6qY2^rHPnT&kpS{cDirtaB!z&DcIR}F?fkM>Zx)D~ZyqZ{bk%+F{`D_7cJ{tGSMhf+%Jt>QfW^hoPQx(YV~*lnoDUx) z1x1B8g=df-te9kd6CBh*H+8*j?Gz!v5dLATn4-+g{W32@KrH|Cg@5F*!yEM+5Q@@b z;@#kV>{A1{`MtYOGb%Cz|9YYI*Oh=BiQ6^`(&j8GgtetZh#8m z(yxMi0(OW0`54f*6x&>r4;+Q?eU={l=Z;e&{i{}Uq9muan;N+u?91ORtrPfu*8T+S zw3v2rd~E{#$}?>En{+4HRsB!)tkKF|mvWL!G~iPAD%U3MH7hbgEWk5P)^(YqNduT}w^BVMj!FXg}Er>$B;Pd2RXZl`M8%$Pn>Z;Fd zq%b7V`HpZhaEYD>`A2*aR3rZIsb@N(^5fn#apt%jcl11^$10$L^p}CJ1FHF36sG!q z&FZ*lLctDiDJ{%>XgRVKOcgkfpzxKe5CE#IQkFXI9IaRb>ApJFSeJHsR4H~+Kh0gM zu=q!7Gh;rFdy50D^hUu`qBTQFd8#3oU7^ltxWWss*_ej-LCD0QL)6wHF=O4XzK;){ z_tWMw-oxZQ6QIL*>-7F*4rZ#-+7Y4zBY=rof7l3u6$MUDJj(HZ_qspfhmd=Hq@m8B z5)@|x$q&&@e9PB_wyqn5s4QT}UkackXm^W!J?Jt`d%Kr_*vq4!^WbbXXChjn1V$}&w6cKN`V&|D}W_a|| z|9$|4%+!T+3Z zceRYKy2n(IG$q;KC(;i@*P+HFhtg! z;1gDcb?}sIz^(0sY7i|=IPx`Znn2ZA430JfIm}?e6nAjdyoKN;AKvR>m;daytcO~o z=@zvXOq#=av;JD(Q8NT`3*e9e1d-yy%j`bJByvmXh_=Ve?8#eDSve#habwAcE%sDA zy+!&HIL$o{NU^(<&5%HzK?`c(0B&;JIH5kCBz8C+F$DK+a%VfN#MUPMV9imrtv|np z877R=DvPl=RXZ(1`I{G;M8 zyrUK3LNhkG zag_2Uj^j!Vbk1Rs=I2+l;ANA-mNhm*#)DFhLktoD%O6yOSf#N*ev_-okhR2dUHxoX zr`m~Wx^Do0(_fYVucBIkz8Aqq%rwt-PO(Z)S?LWpK3OfzJ_y4uDRMBiih7}Au9NUV zqjwzGLy>nDVZQ@iN&7(|;W;zsrs4XV;eNW>8ZXaW!}W#T-Kvj=m%gb{Gj%D^v?XPP zCv&zb@T8=(DJFlqdMISUhJlrz(Po6vmzf5hZ|a+9R#hnj*@9D1rHSj$Z%m1GT|bGu zJp`glV(B-aF-H9J5YbNih8Bf^&C5W}ds{wWRbie24M3(RebC%8B-Teg1+^u2-XM># z+Uw1bNO*-;m`>q5)Q6LJ?KISi6K^6-n{+7iTqa_4ak@>Lb=KU9RJ&R;s<MyI6Vt&G-`FJ5Nc*jE}uyS{TR~LjhKLkD8<23)_9#DyXs}LAS+9cbMtOrt> z4gJjS2?pgfD|~v|M#P&IsFZ{2hGY?;dR31QqxlG0#Q^N#K8!H8c$@TtJNoX!_Q`D>x9E*}pg&4lcIYie2`LAyX|oCD#gEdBLe%w1%@8)+jBHq<-g{LmUUK?h+uI zx80X4L9Q^EBY6tBpgIAD+RPmej2dX)-q>Sx8+AtOagzX!>*pI@J0T9o+TuW2i?_di zQ2-#q4B9JE7-gB=cXQ9DC<^>LTlUXox!)`Ez_z3j2j?UQlXm%5%-)iSDU&Ss~`c^i}T!>9KH_iKVha@kDsqA<+)q9q~{po_Av_ zmPv)dj2JH|g0B#FXy*5T(q_Ym39vtBdGW-eZ9He5B*R8zZ$GbT=@Opx(uGpC?{GaK zIwSOd&7s|qLKl@Znd|F~?o~EihW2+_iA_!FVul{1rEq&JyJ4ibi`UQ_FvDqcgx+T+ z*3Ak_-W@Hq`+Rz}`!oZ-55nE@cR}AbA-@F}TsO)sXfYsaw)ENMy<@fK7nladx zD-?{m4$8_DJL;&X4xu}-D4B)ZbG{ASjY7rn5)_OF>dnR_X?wh)a|Osarb=y=4r_O! z`t;+h!D{E9QyM`V&J{KLIwCNV^soClL5oXZYEnCUVGuo2$(cJVZs7L|+%d9hxV=J2+Th?Gk5$Q7SjTGxy8Tf( zRADPkxn@{?I{F@Q-NU3JlU!z~L&FTPmSLo6|_>QQF6Q@@?!Ggn-7 z`*g^epBm#3R^I+&V`|;~9rW58>e@%gLzufCmXMflUC8pES+Pr8{QrxlcZ|-Y>Dq>4 z+qN_DOl;e>Ik9cqn%K5&+qP}z{BqsT`~B_JYgKi1?ZSb*!Mo@ct+4V@psISG7F`Y^ z5)_U>RC1KMRIZh3Or-Tlwbx9)n46Mra*r~sr4tNv$n%yKZENfIm1-i#I%$XSe=i^6 zY)x2I#uHH2mou7V6k5utm)HG~R(mYRwL>)$L>*P27gs#(>mZ>rGa9LD=ilP~$9b|VG)5TE!n8c?4<1xpT(O{oXC%z^Ah z>rzGX=%?-V!hL=W!KyjN_4`pIaD;Q>zSj;6LjEi!#+3jp&(o&KD@S!wbVC;!ZT}ph zw^W7@vv9++!7x_(7xOw+96_eM$~Lw-KFqcsa)$%gP*OV^47Q%j<@E$=Dke@Eo+wuu*a<+ zimP)ow-3af8a`F{1(#D@JyQNb< zRjW(1&mo8YB0hOjhN=Y=ly8;o|H8MzRbx*2EOK@ZjRCPIIw3d*{}m>=Hp`qp-M|a} z%dLVTXDt>DnCrIPDc2+MU64(`&r}gr0#OP0nzRsgF#7KGv+1bISng%#n6wb~$!FNW zou!#tyoZ-avM#bx!p!{H@02_pJdR}cO^)#tUe@EwrYA92f*yhv7`qu_y(uKb%zKw^Me5=UCGo>tl` zxC8-E<3a2618p0#i{JjR;Tk$AZ`v;E*l`otb0@ecydwNDWIn=ooe1q-@?Sp-Qm;&i` z3$%D8$rXk|I}EX1%yW5NAdmefExaCD>L^V}^B#_$St@%dg1nRViq0aUXVWQ`B^ru! zRZCS+8;Y!Owav$Ly4I`R!J1bcG24Zi!#+Mg&YYzLwIh)%Uto53S7DEV?CAql0)-{GL6Y25ml(PK#&{Fjr*a3834$Y7XGqExwYa!W9;%4&9b|fqRA0%Or8pDz2z7z-S6ueJvpP-Wk#ojf$&o%JP5$1(q zdi%h^&cngMxdy5~nMBes#u_~`_J{bB@{+h&A8%?B&KEU8(B!S=PYn$x{|_QWfJZ(9 zh>3~SeCvrXd8Hzu%u-F0kh*x-Sy{lYKMf<&1wh4??-+aFsscNY4L8p+$1>{U*hdl|~Wl0K=IYQqF0g@G8Fd zbG16qRkgWgp)Kj`>{ohac%sKXqrflw^Y4fk=g!8Lh`ya9>8C_=w~s<|MX^_mmBzks z&V&#!xLyH%h5P_!Ugo{Jo(YyJUT`dd7AMlkh*0!0vIu?0_>P(TR!&~issVYb7<@(Wiwr=|ic?=4+p;fZF#oZLMX1DFF0HSvX;J^btviF5c{tPJR(jsX@F(KB z<|S_{=AJhGmS2jhEl!2TZhZ^xTRJ*O8+gFI|EMEw9$GPnA4P!e!&DK2q~fF8@UVt7 ziVx^BTfersU&OI7GN8R&^+~`8O@q4<`RC-FT%O47h5p8MNI>dKdI>ghd837@t(O~% zYd}D73%@Z>-6WLD6QqIFpOY+Ekzo=)Q&#@1e-vh+W>@f}aTT$sFyI}uyUV^7YfOWI zpOn9ROz*6V9^U_*)KidNE!Wb*In;M`bweyd6&Uo((MGzyWfi_h3C(Rb-ps#ljru=D zk>L}oe~0>yqG(3KL@!;}JVUWmCP?LSQiG9HQ$hnHXSND0k9!Jg6k5klXu{?z)4Wo2 zBYJqgr2ZS=M>T@*2(llcL+O|s3eyY>{tlSC{Z$6YA;@Za`H{@jJpxj}r&Uec3nHZv z2_dq+iYzkVnzWH$Nb)hNatinPShQw00(#5@-m_gGn}>B9rA7%{BJh|qiFZVj(tE>C zWsW?3r7wec;S-mtocL%S``KNC2_n99^2|k0}EW&3fcv6FS25>pBJ*Acm6D*)$zfa8^EoHEJ>A3w`j8RnUO0D*=9PNtiHb!R@;y?@1!QHz z|5&;q1`H~GxW2JkK+5B44huT%AWj=Fr%+;KsVvU;r+5#@keOnerLrWE@e8yp1)2GJ;XO=6O-DSnaQzcibwn)i%tCU7m3ZM>#7__ zvAc=r=(ppVUnzUgq59{o;SaP?9X%Ky`G%m_%HKhLODVFeMR^1nMc^8j1isECI=(^K z+`j2_)~`isJ#8O&m@bByC7S{DxGGg}ej?&>9zLD)u-p00Vfe_yCFguvgiW=xDZ7ZX z@z?Q>W`o6;&5X~E9puHIbYm5?m^{Qp%C$YlAmJ1KnTz)@`$LyrDfce}L?@>w*;loo zL!W&D2&G(oB-UB|N!|Fa%_UP}q4r@6nI^AaZI2qLempbeCC*28QxoQ-d7%hscLwOW zRUXdIzR53ucaKttmmd+!oCNdt&1K<#lw8Bomv3F`@}cQ706uHxujDc9r5PjRd8$*a zwJQ4j2zlQ5ApZ7Gn*y)UEmIN;!CMNs{= z-M5hcW|)g}bTZG1xsT4HerDxb;V0GR$>02IosZO}yTAQv&HOAA)@R>TFO=`cd8 zOEARbh)6g>3cwdAGK7S6iwzl z$z?e;9`aF!<#0pj_PIJgmK-F-wSVPC{w@x?n83WPr)+3@@_8PGavI*hVaMhA%xl=` zV0;-qf?%@sykots>;*in4BXOentvZ`^pkw^;r*j{^958!1GYvN$N)Pak!)y|)jYB0 zvt(rq@ehr^9HSqWQ?lrdN}!K#zZe(sJ#lscWlgD#Fc=Sj!7oIyuZ4q!6oBpio68#@ zX*6vo<;^zsEt1-t1CZ$R^QsXdadypD>KK4k`u^vM=XXQ=3GN<10_QI_Koz^^Urfrw zP45B_pvQM8#U8N#iTIT$kN`l0u>PKj=Dx2MZ*%K=yC|}H|3;6jG4F~}xnf)0L+MXT zNH=ag@A6KMm1{Ne7s~+^2ltpX<2V2kJHCj62@)sjP{Zase#siX{@1lcFeX;|_=4^2MLnZ6|_-X1@zh(UAX zDAPTW`#G^i<}h*qUs?8C$iCHm2Sz!~uY7r1g_S#%YO1lWgvWtQk$B^qVnoh1uB=wm zI$-0zeW+D&w)i;9E~r%$d+W)8zULfX&|U!`&%S(rR+!^Am!S}{301qlO9(_KxlN$l zdnLf%ghZSaP7vNdIq&@i0X=>HDHSV!&S&n2vSY?!$+*RFgr{#)qnGrs6IIqNds(5> zr5iqkIN5zcb`0R&yHn%_{asG7?pxO!#DOefR0vfL3k+?3@-4J=TYB zE>FG!DRh7>dH}gi#TVbzPrCr|(=M^xOj=NwHvppr02?k=^SdYhWa9`B@=J@jR)-jA zT8I`pdsitLkf|hH?CmwK4XJ(#h;IZS&P>N>0eYgj!_$VmI;4zJU;iXv{6_bx!2>El zeQotqaOFG`{M*N#5&U8?NMFLjmQ^;8D%uwLU zrDKeYoT~$kA}!Ic$!hOI6V`HBiXjC0*f&n0!qxjm&ijjjfboWu9_`D9=*Fq65pa`} zRs0MPGqe7VjpmlfbBztTxqQU;&|OmDaYvEI9B*&F3e@4F_%!r%a2Hb`%EVmyW1TgD zC$bIdPyi3$!Eq6qk3`lPFGk8T9;PYtA^)-FLz`-b^KV93F)DRxzDFt&+LMW~T44!ypyXnO{9IDnU1W?>W==9S z0Tk#?%Mi7r->xoB|F{8Xge}iMqxJk1%eyD#_^PhPGw=CG-w@P%6h-{Zf1+4z--`qZ zj;{BE))N?OGBpwC-HeY32izTwH_?Uvu{n?l zXn5*>ZL#EveiS9Ou8od%niRECFQ4nJneKz!CmkyltX^L-Vn5*2ad4vCw*nJ-O)SxbvTDh%@>dl>~ld!FLq z@$!3!pM2-Zn)zbwpjzzC+K+CYgla2m+f@JBT0jGvuo(qqh#Ibdq}62jn3Gvqf={Zl zZgStS1josP z10??CTXeD@PNonh|DmWS`6KXsnC+6rbRG(N(d()|lcycn%PTbfFF(^0?-O5vTlXJm zp6|s>ez{@fvtOf`ubdR`nq^aZQ9tTsQbK0Q$MCOB^4?DiGHpYHAtY1QN0`49(2aK= z0u?uNf@7tB-ZpU!O--Ps(>0pz|>9IB+_(&|N)Dx-; zz&%Wvf4d(5|MFULm=0ERf?COxc*dmVRMj|J0%eO@LAcHo+3I+(1+Jw@z4v>@sI+2F zb!3T;c&ZwmK^3Yd^w*;@3D&h2ta&Gs5Jk))IyGWt5KDqKU4K+aST}|__NVTU=PTT4 z@~8Puch!;HgYnPBr~g3T5~lDWi80$C^@E0k@~ge*ZoCEt5nqHtto~(o39_>?QOECK z-nlWs+NC1{^%4O+E|Vw75sv?%VG~MYpbwv!Gb3n{{g1uo|FjZx`x{!w_ce^f0~zSk zL%5y+&ayYYc7*4}kLYm$<|nt=>VkjXL-2f(W73>p(>(n_v(9b5Pi(1!ng~8flL$LP z%p1BY_8ZSi;05g+6)by)!0)8{5bW8qo>jn-d5X%V$1DDbP7jkiFtLzGZR13%>)VDp zLx1>8a43H;Uw@H89LguulP3bCKj=~8RBH4A#(`a8&+g9)Hw&&*IfMZdZh(*}3l+vk zNeule2N3?6^_`ekDBbpHcQ{{%|Le;Hz-jkiVmcL)T2^7+t}o{aKY_XTm2iicIHjT4 z+Sxlw0QN?$aKA;{nYaA6Vp)HB`6d@8;fVPQDDPP22|_O{^z+b7nHHH*$B`d+vuDQE z?;n8St`W0mBqQ}kr{U8CIV$Z}p#sv(e~KQ%KL@D`C(!~%s6}kHR#^;P(bO$mW9x}Y zebnm;hu4%i2?b-oav8`nfm)b**!GZ8W9!#R`DX*6eNgu1oNgkUxH7ZYys27e1<13& zLx`(FsE3x#Es(uQjMX~A2MjwpI>UO}U7o02VS_cC*6Bad-08+=_PL(0#!xtQhDnTL zbYz?k>r31-6WE&wfpaSIj|Toe*n^g=&>4mM=cbJn)aePqmlL>)3Ui!%KFHyS0n@RPW!kP^YdO`Sb3jJb#B(&Fydy)Z%s6WRt9zxeHOy z5rq$gnub6nISF?A4z)YPODSvb5NufA=xvxam6U&p-4GxSYy?PQ1;8BeCA{r~PzL@~ zQ=j67)-5e@4c5}SSPdXe9JDmuWrjeGkamQ;ha}Ht3z+tX_7qR~Iv_G3kM9{l@*JJ= zdkecY9_KpAUAXx*lcTSS<4A?zHg*Y}eA09M4hW2r%=C-=^o2ANllZtp^;8avAhyb9ys+9)jr#GHv&;ZA90USU ziuEC=@cM-7~H(RTAg#b!6?)~vk63S&b$-2U_D z>w_A$1L-B_t(U6M`SP^D=7~-It%P0T@F1S}fPT2}fOYK2syuNU2S3gvnZ0#i4p+Fv z$y-)Z{Y@D?K#ty<{{UE6ID7-#>CavRSnrh=0FrpxzmE-9XoA-y_Y`^TX2*Cu1O25D z6WP5T|ECKmOKhRHBhEn1A^u9V|4;FMhMA(b4~xiRVeli5DaU0v|L2HKVArJFuvZmC zl;<(xKjzq*QJ`{^W+%0H2gV&&Q0+AnRPMI5-~^|xA)s5b!!KoxYZS?$i?W00T6yR_ zH~uk&`rO}}6!9z^^)*JB2 z1X#1kiT=|2n?=6=c4-?zR2l7`gaCZF5E}_}RHdWD@ax5FbcpN5RCEH>>zaZefxw7T z`t@1c69Q{aNobmkXiPblBWzevI;21Lf3~XVJ2Qp3fhe=(=5LIz)3v9|JVTS=aE(B; z0U}{oh*$iMJwqBW{>K?X>c6{2Ix-YaXLg)oSx}YqPArJV^V76eHnhL{P?i#9)(m^p zqp(U9=y>WRbr@r&)87T70+uU4gUsoZ0sJFQCgiBJe$6?u(RK7Pxx#EJR%;e=sY!YX z`<6?cfyc7+o)+djd9`fsJ^y(P@{-c;WM>As!1O+hmY9yW9RZP~&$)Mw@rNW)v7$Q;osI7!pZ11S5J-Is2au*-kkMqq zSeig`6ro7?PO=0{^F(|=Yd{r+5BvN^l~~ppH`|Yuen|3w_2bKlMvC?qH%oxTPWS)G z)l3x{u_SI&fgMgF6XkWx09$1v#ka%tW$e(wy-vOBzPUiw&(q~fPtDqvnr^)y+WYOf zVfi+kegDzp*J{gH@W6jcz&E`mVZo*U1M@x_$2|f#Ijz5XSnn%q0TU!)|Ag`lb?EKH z+}}S{+Os@HB-&J~t??+2Xx8B^x=hUehI~arhINU{uh}}FZ#v{ER`$Rc-agp!O6OvD zs=>Fx*{!WXKlJNAT!-iLCLODn%5CQ*MOB~F7zuy1Ptg%#H z&g*o!<(mAabAt7^h|fTMZ6R$zcKp{ow{8~9xGFGmE%l^IB}}<#PDQ!b%Bp|mZ*3(3 z7Mn@-dtYMN{pv&$q)JZFt_pOe40{4&f2!sf$~(avj(pz7{6He0{Tno$TcXSv%-#zL z&DvBMf(PmB?c_$x61(K%t^pyV2;_tlM`-=8FJmx^w+5Dqz;t#Es-^iq9!pD++}&}V zMfAji<(kjN5Jf+#YAUWTQ` zz)dD0o`F4kmLfRWvI*TeaaO+;$F&KOt)8=EEQW(urbT7RCkU749JQ8XZIh5i z$vw(gz?39n%vV#z=V-F$HbH4ZY2NCyy1_bt+WfPs4Oilc2nSzK_AQR;fC8M!h ztz^h+_sPNzdUCzdY`{1;E7dj!CO$YmLlhE5XuTp{6|4lhT$242Sig&kfttC6Gud9k zn2T9!R~V+yvLHEVZ7_{V*4*51k&ot}$l$j>VtijR8_YfmLcBsXX9<-}CGQ9oa0lzj z=>IzQzP{A(tgJ41*oB3Urc63u=sP?=3O49n6D#|e{l`NAoeozO*(N9j+qF)ZhV7bpUNwWf3;gX!R>&H>1p(9bz=rt zBuWHf;`l6pK@xR}8;1{wj2~^BXO#;8#4Y}JrjIhXfK%=J%Zfs{L!AEKy7d0bt4;H@ zHVgByh$FuIZ_&nKVo(pk97zzYrmthI`$;pu`AZC z==hi1|LL?r+Z&5tP~<|jrw*F^>o@mI{b(yM9k(s{%iy#ez!+UP@08|zZrU{^$biU? zI)FufsQkJL%~=7T1YQO`IJn;!k3vM!$fJKvdcVKybJ0%<&Tc(AZ2w!9WwtmS#kYy1PSB>6>@Ljt&I zQ2q|>v&aN^v&cCEdK4|NyP+w2mgcdkT+_rROPpgt_{X7TwKNus%IqDdeclI&Otlma z%GSC=5|eQaG@J~J5KPql&z;P}+sqGi>ZK2y+Y_4eYPJJaz|GLVYqT|RW&n1Ze6q*B z1ka&j|I1x~0!Rg`0|n6!eq@7lH)>#a_@mYJd%MDrM?nmbHObikQDfggKQX*RIli)H zhlNzZUTM}>dedr?%R#ep+(Fwl#L5?0Os`wFPKEU{;Hm~qIOeKWB-R&FHy-5%*3Ku0 z_%8b!>drNoG#)~MFc{ap(}Ql|q$s|wheELbhrP*f{563)FX4t>NUo^#GCCuRYr39* z>ElEXAMdn}`W4@QhsCvAL~e4-sY>g}Rpki%;q1DU7m#4I8}o7vYHzaWVm|l$6xmOZ z9)MjQ5u?lyfTYfJjyDgzUxX7A5mKelA1>7Cc>5!#LPLc8QU_0i`DD>xx{+*FHa9q9 z7yKBqbcmWDgLJ2dVqzA5#rvzpXOZ*7oBZVKE$5kR%^&|mwdAw|Zd&=wp8zu4w1a?= z0Rs=-! zz7CBUVqvh|3?Uf!9ejonGYn(j=nvcEI=l&`riKbY5vO7fv7b41NuG>%?Y|T`W-X0Ix>nbU zCl5+^2K}cB%lRX)CM^(rXd(;YDoWg3SO{b9%w{6d;2MPnn;|!+Lwp`k_s73ux9i7KUcan)5mF2tS}b1t zyJ_|>=hz@K^SIhP4+^ngvp5V&x{1ne-viCPVcQ`zYEcasvuCxWAr4I8QGOXr@f(p^ zzj9=j*PIIY$X|Whz0@Rr6)>48P1ohn6_c3A!S74gD+wT1l#P)0~0jnFXKD z&-Yqk;6X4^4g;^@X&LWr*m+D;wW@jKzgu2W?niNA1O* zkXY2~_rtUs$fn1=wWW;>-l?&v!RhZ9T7{f|C$U38`>b9Ja;zovwD171FM2|??@A(+ z%=4r3pZLUwec@$gV>G{fNfF6|0#_98RyRI=4-*)M(i<-z`!AY=@;!d*4+6a9tXXWa zzcE2-`WQc_Ftgo=<%*iOlAp`&jH>|D_onKmCFaG^f^RudKZ1KW$SS~(I{1g|cZeyS z0ixue;&l5*LKtPwC8;r1W9ihjmjTLVb_50>Dj~9#>SRC&-@0v zm8=2#u5X9I)-RHJ<3fg!F9|^N_~KFEJ16v3L)sjj1zn4?aBAJ zYE=6oWD9ol;&XbZQTs9g=y`$r#8;F)hU@fvNV$6Yp9N#*!T_^RNF^qL?>QT|&IzGN zwfEWQYb3N~h0Rmxbd&I!h!uG;+5@zqxn!4X(wSvgJ3^>wb;Qjo3 zPSx|hp4t>$SMUA~9We_$7<}jBwfgd9d*B95RXUZ66Ql6N%%)T}Dm&&>LYHDGHl|oQ zbnj&sv4bFaKzvmx*_`vZC4J^*6a;FgcL3*#=d8~?+uq)8Fxy>7-Y4U)c*@-Y;5+rz z@?E-qm~HjxJ!kWHLeL9ebydGJyc+*;-Rt)wegByFDbUTu$Nf*AH!HRfc|_AK^dE^6 zd!XPBpK(1`=yG`=%s8(f9RF@U5VX#)233%ha}p{LPb_x;hmbO}>>~#e9I)9md-~Y} zywz>g+71YH%R)D4H#mU?5oQ#|awGXn z$6sX%VX6vSgAog~ye|i4l-B^wT{plnQND0T-$g)S9gacEp5c-;(T3vQuO;tap zyxo}X0;zpRNp8fsmvKB#SE8MZPW8PTT1j4pYXs$*Qyz~A{1*i6sWoM<_yd_XEH7Yo zL*3nPbkID~---3$AG`0iwo?dkCixJ3TVKc>0*}CRQKmpSf;JfjyWe4BA7n^P8y@bI zSYD=(Ly#Sz^=kV?=f~ZBNXi#LV?Wjj@@g*f8C?T-ZT55-<- z{~in+=0hv;sEAfyyOJY+H%ofV<Ajlvxj8bRf7iw zoRWcR>#~Tfr&+Z~7b7E5dBhoE*UT7?kV0jQP;2>58mAgFjO+dWm6yaPJ9#NG<_Vnc z7^1T@R2%wJ2}lIiA^|}R>H|Rhl|l{+!7EsOI51yMsyr&(f_SA{lDT{^JPVpX8>wlw zOP*^hu! zpP)4%qi~OwWbdK{({{V(YHBlVZ%2j-Pl)h&P@6OEnZ&wz1gUTxT9wF&GBu{D{2Faf zOsFIU5j&8}gy&MU_Zlp3JqShq-j5tq(WoM^5LyC$5qiNBi?N|G&fyYat_z-+R>IN= zJGef68tSt$B9M)kgx^*9p6rk8uW(%0xWEy0I3gub%M{ATFuuCw4w75!dv`=A@5w0P zuX!O48+s8>tOc9u)n>%Uz>^v5A$S~Y}>9-Xjr zT@9B5m-LEhWc+Q4#71uWpfoUHJ9#wZU$wadOS#6>G^s0=e!>0V-b_-%x!jsi#gpK; z?NVtuMLb%+NfhSiEM4<@#yDCJoMNt@J$JLzJH3$gAw72AMxA{P`v+>RPl zWwyuvvPJ>il-*x$m=JDI2JBGS)jKO&soPpE27BAWxJj3&o*Xt-sPQF{U=!i+Gy(>^ zlB-?!6kqudXby!Pj#3dK@hqj_$ zAnXvSbPrQMsx$}=Ofa0>XaiFp7c7;7z(R82eJPKaBXc$g#^g6F@OzJ~&leJnKX5)A zOLh?%65p_ykOzyhfwI)@*eFet=Qxs202JM{PwrcLQ^S08=FetsKdPXXNj9jJUbgzT z=vq*kq>47GqkUnyPwOlA>Gjv`;p#?Jxk8(&33>_TL7kheIM#=syjoW}2?JCMsN{xk z5NvX*>|2)nVMiE=8XOK9XT7|9>1^mPnMGA0p`cXjTw?;8!%&afGSVYWs9t(qe)FIcPhnbwY6M(&1+X7 zUDp#zk{u_=MOZ6EnhA``tRNj}=3}{-~)fF070uTO^QQTkP)O|iQZqwpIn^P^9Y!qImncMCk>czW!oM0=T&k^p)RDq zkH>v~jlzJ|ecv%Iq#*O#OwVt{o_12CY^E+Nm9(}^*CZC{u8WkK=nWENLt}$s3Jg>Q z<3Zf?=KqYJ^*oAjG81(mm-e{VKI)P;T@(edNrlk7QQd$f6bDw4f|-;`5Tb8u^eY*L zOUR|3bA(B@U1fuQoli?blK|^V-jS#s=&}tm$X42$h|GocKQILP=n+4sHuU_X_XjdN zU4ay)c2Teqd!$h=(KS?f9TkWYXlHEhdE{7x#bCYPcw*Q+(s8<}Rmi<+?pvu@-m_&; zkep6ZX&fQ=*Wv9y1_cu?J#5qvm#iQ?Zc%4mgzr?`;e&M1A*!6umdR3`@(vX|l!~AU zNB+m5P5JdJW~XS*Sf_UsZb&9oGZs-MU(+-**6@kQn~Io?(EWI+t2#;bxNFE*_>Kwr zQphf;uud1pZmBS-7LLM2VDMx$X9HvZR$qxTY>`tmd{X#?Q{o^?i0ZtZw}Xp~3qwTd zmdn#RIOtralCffXO*Jm=-e4?Sz<~R|~ChM>Gkn+Xo%*DB9uO33UJuo~R+l zt9N2B1t(VM<^8L+rS3{ofwCW%4UbV3cx<7rNJQOju(|V}G9UA}aDBhiJXN6U9MPs7 z3`Yi)g|lmPlt2Gv*b@-UuiJvk#(VB@Dl&)z zS--~_M!8)qG5h3?1(thTz_30> zrzBAHr_$zA&9E}%hayQw*H5W6u1vTZ%gsu4@*AZh%&vTvdYHo;Y4D5i4tUIoBT^6o zTp~H<-{;2>PrbJDhLpnCj|6*yzeNWz(owzDOW7lJ)Uv3xhBcX<70)`_=3@l6No*iR zMIo6Xx$Dy5>1N+(M4cd0*2%_@Q9=8af8&b)i@?2UUo{G5W@%r)hq)QA;fTQw$?C%J zI*owJna3jE_=`SU#ezzFp*Y_5Fl+-*o|#km=}09vkwK!nl&B&txe)cK5RKKqIamFf;Xe!u z3*(S!f(CeUo1nUC*!Tr$*_T7pOM7lv2x1NL{0YQ9In366Ad`W zjV^$qj1)4)mJ(}KhtJCD!4ThvZ5haVsmTWte5<&>_ns z^u6HUfWSS*BY^Ls?TzpH=mVVi3paO$l<3<7UdQMFui?DWON}94X>&k56ik;#a^m@R zE~F31qi$D$cqO^)={`{FDX=?PoOT51@Dg%m{cD&e-?W)Oc_?9{SZ%JOGfrM3lg4w^ z%rTmOg;g)pOWf}Y8Uo@(hb`}M>C~5xtkMs}I;E^W&$4)7nCw+_Mq*gfXEz`$#R!$w zD~$K>Io)DQs?~m1zfNh6VmLAHk+5(VhLX`XwM_*Gpvl@;@C-qQyPzb@EPaF_f96!n z6GSEHn9QR&EG{mUN{Ot?pb(*o+IgttgRBiufYyTuaawfz#y+P5Lk+3-5Y<@tywFl) zJ2SfRgiNlgZd@Z$FSxBiW#p^H}6c`_bxIQne*vXy+=+Q2+B<%;Bf-Uis?V17O`lU0`3=_2?c#%aW%|&>r?A&T% zcltUd!A9gR{T#Y`6rE1=p(Q&bWvC@+_EBp@di^UBp-9)&mZZu>1WDV7ej4mih1J4H zF5Bxw&yDjE#M9}H$y*MxT2GBo6>R6jh_UU>_4PC{FDb!#VuDI3g@chnbcB#;LtEz{ zuZ-&4NOrcCw6T|p8YHWzd@@UnxxMX;YTS59$L)cRFv-47tjtn-V!Y;lj+Vl?ot_1} zF>B(s(R2PhX#q$4`Bl+8SEBcCiUGM5TVHoi`-kV}fuWC<9Lh6P_bRBbF+Z$47 zrQp_OG!QkBk<~sT%aJ<^))#gJZezuihMzvL+_WEO5qDrMiS$JV2VtVq33%qwxt(Re zMz@8ll%&jyHIe>{haB_^hi^2|ZPfFRLWY!jkaa`IhtlRq@~98E$56A_G%7&aS73)S z7|UmhqpppT;KfL#mQfrhB5TKJ2$RAmYWy{w%a{7?5^wR%%^EgrS#Aka~HggOWo%LRtK4BgZ(< z){yg0JOx-vLnS#v3+dpiHGkefUEpdSOmFv}?Zp;)93#C5(dL|89Lw4W_YfMj$v;=n z3E6|^5azix+49q8ydGdIuaJ4e`%Nvbd3uBJ2J4B(Mch}OSdSrHKc)@2TgJ;L&o6I3 zZ#A4j4Xj16q7C>*;Ae2FVC`(&XsUW^J!W~uPh}R6+)cl_?0x{Hr0MQqnSiZx2ib?4x(>AdUSyl0RZng!=l&$KpK+lI6kkm*jH zLEjd}7Liriy{f{~m^?hp!;bLJ>6ms!GZ*NWR?#MFo~PWDD)5TCb;8zb|=G3*BcXzmJ)iw^n}- z@AU@8H?4lR^PV2zi{PGbWiUX3Zf|GjiO0uxWXOx<^#&Q2B#+~-Gnpn@gG`@MmZ}l+u0|o;8GfBe5IWG(~ zSB2yRjhn56g$Dc)9W7n-|5_$5G6COgscftacxz5hLAPU2)`ZY}MHJeL^&ZLWSO=Hh zLTr|2Vwu-)4SFRN>~;x4_@YmfMUyCn=6{(p!g8Z?1S@~ULwvXS$pjP=bG8P}=2Wbc zbIX`MY(6bRl_y=F=SV4OHo^6wsCQwD-GYN^N;CY4fTXD`L+mvG9gM2nz0P+MM!!E) za3&eDSU#REb9FVAZ8)=lS%Q2fX-JZKJy}{|?w)Lya@OMvf}tmZamGK6egC$-Ef*Am zNnE-}yEHOQriUjYF&qzDsMLBAFLvZe2|Wek+6~DM(mp;LKR4Lc{lfP=)(MsitM6yq z6ySkC5VD6!gt=9P8knDrY`oeQ?8qD41IbLZ9LnNEzc98LkrR#rYsH({=0?GWstkP#~ua{Nq5|WAx(?nMzDtHUrmEgcJFm zL#W5=HWdy~4^71aE|OU5g(E2=e64IO6N1oEGcAyz!;LMImR)i8`)_^)Me7CX_ELB7 z)?EI?4@?eUQ0*Lxae`x+N?p5^)@u`nmP$l$@^?;n`9gk;USz+vW@s;WS@%`tnBz;M zhJokzQqxHL<(%D$>u8P3M&%CDcSrD>i|gpmG^@hDm!z`w4BZ5~=i)lwSSMZ?@!sG% z{F!I(2yMLAB3y?DJ?6=|If~Ga#}P#VVxFF!0RCR`^WnT1|5Y|2-}orRFq`IOBL>f1=8SrfV=XnM^8?+|;(r#-dn%GT@~Sw=5-;#dsr~}HBx;mzMh^;V zJ1CDoNByo}kkl}!ADJCc)+!kG;M7lAruKf$}4tlH~!(XQyd3F}ObAMLJ9i}UbU$W~g3MRrApIIw+ZU)*OhdvVi> zVPAiMk@|m2L4`=A-%E32ShDEmF3C~sU`k&g?+e^EZLG0YQ-7fNRs9HH)i`2&Nz(5$#uslDUF{$PcW)#Icl?<|vuQbrAO&|S zvAFP76aXVn`QMOdrM`KDkDinkU;3}9S!@=FE}Mm0aQ~aZPl4#S+bKS^KMUm|Vg|pL z*1LfBW=jp>&v@Fsln>0uI`+>$P3=%{W(-hRKHUK{2nyafK22^^Hb zgFhR=xJT(G>9%W`$k^D~uM}Jd`XplSyc`;8(Hk0O7+8GB?LwE>`z_)c0vi;f&WeUo zo(V(=DDu#lbABU(kUXRv5ZcA2XLBI>A$nq~10Iro6MaZbO`qTF_^C?WoY*?-@+c|m zVEPvm+$E4K-?X5_zqoKT`~90w&CcegBSC4keGb<(A_+`|JjFu*6!CWVZBIienC&=B z5*%o>=Dc^Z5aNvb)fMdKJWX0lAibs{1PkvUjQ(6Ho3rvgQg(o(NlNZQfRDufP#4YV zjploLr9QfZN3A-UMFEmtlic|fh?ldr?eannQZVoS&FvfJTVde|xt_vXf`uElLfieFb!ey(>Q3dY{)3@(0x5W=2LT_q}sSYc956?ud=m&z}{yJFKSnWC6J z;*{jn;mxI}cP%*0p5vx0RvS8R0fpDg2+MHwAX?5_CP>N9*N(iM_2~|l2c>L$PkchT zzftAynE-FI6GYrBwRqR4K)|S=!mr(tDi4D+vy0Ykb@U)}f(iy{QRJ$TYP56id$f4k zbHKs*o9JCX^Vc?#u6p$Z?jS7f*=xvcKZ^GbyRLfv4Y}K1f&j6DKu+YOVqX5yvOrhg zQI7cRP!7<0IIYfbdWOhm^ddZ-OO)uSYdkFXs!+B9>;tFIuWIufq#Xg^f)C@#@EDtC zTXDDOFM%tF1_PNR?#&ZxlpYG)%^lS80>-&D^Q~AKMVd?LxEVr`O7#{c-33>0ng#X%3Bjjx;y%Ea6XCK~1%XWJjeHt~NP-$kZZ(LI2SlMDg`HpkG#q$AKIe^C#&iI_=MC#&fArohSKecwDd zS-kI4!HJ5?YFgsPov5$B#U%(~A24xO%=>3H12VIxW}bO-i6goEd3}zHeRv ztuMb&Tg6_s53&+S2#pCss8cG0+V##OHLo|;8Y$B+{!`dU#j`^~*?SWC>^vj^9Eck!<)I4uXt(ASoCgyGXR{98 zGU?A2_uEiWivUhlAGuCbX6hsqlo1&U`p9MQ!6TQ!tLgva>iK46@Dlyrlnh>~%bS$J ztLXTqWbopB-@FW7yzf(Gu!;g}=cB+vG$Laqih2?DSPlC?i6M~omJ*Up#qfg=c^FYZ zgb>O;sI0F^M*p3ul5G@G2702TZu||>FO`TCd42W!Rh)%@cy6{QBY3x?4UatpM$nI- z=iLFvL(oSqhynon9BsHL@JK8oeRRV|$nk~=3|h)jzFp)f&jl|nbw}6HYMiIzPmq8^ z4{6&>zf@OWe+PTWdnvO&+zhcZ1kiP50~Pg1Mfzs(&lG%I`fXJ?6cY83oe&267d1ut zvEE&Y^W6;zuK|geexXNh0USW=R{{226d>P4etcKNa27YDdMiq3#)f0)g$TMb^+wT{ z#uye-M`Aib9C{_89v%!??8YcpHDE)?G}QrSqPaBfQ?sia+FJQiR~$&vgm@gG3cx11 z#`!%Yo`=Gku5iV_DZ3p8}L=5K(Wl#7wAr3lz1f);|0V=J`<5A zhPSBdAqC@Rz15N)3C((|4MIc%Qov#sMrl%Y_2q3V%}X8v9+9horY zb3PTxKwV+Y2S0!Ocz($n6@+OvTAibbp9~}4&wjM6Kbnwi#&4aZCQwDJW=Tufzn z%{g%|krRcO-H8+~d?PcGn)HwZ`GR3XbSL6vToi;z7AZ*+zgcg|Kv>;Vx$X)c>m>Am zaU^z8Kll?4^nu(M)d>lWg^NiK4I%$6IrKeN}oAlxMm%g=6tg%^rotpMInIH9ZK+ ze7ZOW>x>@<`Nw6DJ0DQK9WaD^@{u2}FG%I;96}fP1fYH&IqWp{rjan^O&F4p&mzB? z#AkgVE{eOZ?#hLF)}zR0{2Id}F?%1gwf7l-AdGs1T>l!X?YUsw@e^0z22v2Bq<~Lq zNhl|sRz3l~T(46!1o3e{jonfSilnGA`K5IICpYS&vV(VeNh=d?t2Fg)nYw_(J*^SY z2#iKSELU`AIkA03%POGQcaS>5Qx+mP;t!9Yd`CU0{ma}9p6}a_JS1&3{eN6N-|YE* ziGFYDe7{teH+jBaMaMUFzF)lWn?K(#-uG$e`nc)Zvd1^r<|#<$AAF z`BbU$hcF0`AD`+}Cd2?zl}SjM2tpR0H+k>EeR3@+E@YIG`>@5kqRe3ctlSew9YTdkyzQ`IFDlz6^VHMl0W`l zJzcI?Nzji>ZOKun&U#e%;i7l-aLX6g0)w6d!Wlz1(Rlq2(i7@~E4^&eqnigL#!aQL zp)z~5gxTwy13Mp%P?~}G6<4srTgou`p6-0o`A9izWW*sJkP&*A_({wl=1kvL1n1F{hI``A%u_Yv~hzHYS^x4QqG{$wqGbFaR$^A&s*nl-ryzN$l# z_Xr(=@grQ9Cxh4cfo9lWq z=Sm0PYQ{2J>M7y`c=1M*Y!H3uFcKoS4$eH!6mrQ!@t(!+UZXp6iKa99CR?G0M|{oj zKe;eNlu@B{;+w3}!E0@bP#(sEXetGGODMoqnqSarB=+&5)2@h^Z`3=A0*!j7Rllvv zL@q8tR0I$4Ssg)!9kE7ZMog43(?KXC`{6M=sy`ALch#m#e2I6`L=j(r^+{F@NOQZWXaKN zvgBCH5-wGqpDZ~zd2F&|MLpj3zCN%lS*E8CCQFv<N)j+NuJIkhG9}>Um@U z0zz3?T1x)Feyh=H?l%tho9+F4*J|ThtJaWND6_eqwL6S1>)=8<>=78c$|$JZWRRQo zuxlQ0`G;WWVHBp0bjnGm*-4SEGn*JbjK%QLvlPSY&lK@dKC~EKOD>?T>g2Q3LrFoFroeyT)ojdXFeE2MP=k;gQ+0GtsdtV=VcV0`LoUQ8Q!|u-O z%_p-xy`8!{D<`IAJGDEvXWN|*;@!FPu)FhiY|iUXG_yTDesB(5X%?K!qF2sY%~opB zw`N=P?Re22Jhw%!VmE7i>?>PGs#ceB|)SXomRS9T4Rt#%-a%@F2_R z0hISqvzeOK=4{j2il?>Br!_7Ha)a4WPdoGhbY09a@u3H<&Wwd79kJ+@m?)u&iQ}Q3 z>k?_NdUF&j#h6z!vfYT_WF9>a$?cu+BpwV2Ws&bAPrI=TFLoa?JVxN-yUTciWdhOS z1u}rxN3L84<3_zLi_FKXPPqg$Q~q+z+5F|&i4CY#xab=BMRzXnoLy(hI%}-}8K(t= znwArDN>8-Q?l7)Sms)`67E>91n9SCt2y0eb!<$=%LLYip>3E8=$1jl+B6g(|s8XQ| zJj5vAB%^=^$ie+PnTA0;E8Ne;)~NozMk1y&q9~3EF;A!HnwQM;uaCHQN^cy28yr?hxto0Sd3Fgqu>els;X&dH3e*@=f={+50zN z=BS_qYBl|pqk5;_h~;l;yvOVR`X}*?FY^2Ev5!a4OO?hlXsh3m%UJS*;6LdShAFse z)j6CqxmaoTkBlV(H6s}*Q1m*jf`l%7Ir!v82Le&Kwf^@@Aik2?|CNU7u;eR=Z@o&? zQC!ionhFkxKYNv~pH=IV&=G!Z$?6JGO|p;{gX(i4B8n;g_Z%(8KPj*pceG+SDm{E3Zp`_ERu!) zrF!80J!H;MCiplV!Qc~Re1rZX|GBReyX99-D|zBozEy!}?agfvA}X?FoPPR}{o;kB zDj(FYkG;Kbr?MiJ5Tm0}#9+@uae*;GI!UV`-_KMY6XB7Me1}9nLm_gj0IFFUV#-K( zccqPP!r#7!;;Q#9VkjJ8AE`7idhznsZ%BB}+Y%vZvbm6WUJp9gS9*(3&!-;Btqh$f zPvk%G5nrd^mAQxR>|7!QKD|;Y^K?2KnTqcRDa_wWVb5nHHURR^)#*?MA_>s3FH&4| z`HlVWoTR-P8zLF+SCinNhU513>fj?0mPQUH;?2G1A(gF#O6cPv2C=6Mn8;`JG`|Az%P6z9LhxArUFpU2qc5?uffX}cYg$aCXm zZ{&H3`y`*f;fvUZBas8a;~PIEQRpBY=Ol+}?Db!-Dxn;BErD~DtGH(w*Wa(;7qbeD zv7f~cgRkJK;y}hcWHec$ca)(~MTe&u8BjR%xgZ{*b-Cm6ue_3bug*rn9|5mq798=Y zkOzETHE`lsnxPOnOfGdp%DuXvRkbkQ!+=%=!jL$S^3KX6Q4_ia2ZIm|AVVLCnp$z4 z_w!8On$T738+nSZk7@dDx&sDZK^F%6C>{rseg(e|ktmjAoE$#o- z&X}Mwq7qXPeARiMp2y4Eo z|0Yx98%F~Y}i(l}{MjsL_0pM>axrlrOeg1_F1dmwjHCwifJiV#R@genG4 z<+IZo4*}o82vv>eX~7FJu1{#kTudF2LF=5jsYj{F56S*#roWO~uGj&-f~!9CsNj^0 zU{b<}zoWQa2tF70`zYjecQMCry-13pF^1riUKc?`he1eg3%OoBwC|#EhR-=8?FZ1g zh66FB9sBQ(<4)uO(PRE zJ~3+%9ZI#5N_eRrr zlB>;}Z`qkkouF;WP9%xmX8*RJ@si&S#5~RZ&A0x5pBp|5uz$Pwr?G6M=JRIe`1%{> zxQl-kF~^o!$o02iO%HmzkU7w$qw9LMOSgpJwMgViL1V`{bR@j+a- zY55qU9y)VJ56iFWqtiOIR=%CF4{PDy8Na<+PMY{pd9ZsQc?<_$d^o34U{ZdsGGm{RT<+FIWQ z5CjkleY9zP0RaaR;%MuGpU=|T*ebXgVo#prWoc9+g{hH|S0&|?Jy(AK2;$@O9dIMH zrzCSz6(~6$GHPVGOeK3k>9mz9r${QfpPV{zA_u>d@Va&ulE_!%`-e|%eDIdih|i%o zv#SiI#0No$$9x_AWNfWFJHPW2>`#OYJOeMt6Nltwf#Id>+l8kUCfx4$-Qq|MS29KTd5?uU5@uoF-FsPEMvMS?~vPSRK4gsSi0| zgwz`x-^t{pe68_yRdl?OI?Fx|(TzC43$40=oZ_Zc6O7~r!Pl3vG8kE*O!cNp(+X}- zZ%D{>vc}};8H!2CkXM$Z75SzUnx@TYsa_{%bp9nV%c&X6^j~)+AZ}f%k~ou#JWxdt zbI98z{Nq_#X=s`MB469_R`7}6cZow2GycLwAIQ683jVsF-VC znk<-Xej=|ZIMZ(d{o2mX?->P2YXH=-aeODalOz`_CV)BoKLrjBSY)TL|%6RNVDWRoeSsaC> zYC_)RT=((arN~~2Ih#_7)qcSDO=oXOB8S?E$4q7M|Wd>%U zx0bCk6v(5i)ZS_Ngk3ZW2;(0R*>mE(APRl09@kS$d|9o!-xo`TL@ds9EsoAYi_AND zP3t>5%{n;aOF}Eaf!sDiGR88aH%wE(SF1wE{^Xc02Lg2_SUzOP#B_+|8n-9SoC3+n zgy>;vdXzy3rM{`n8p3+5tWctt=oT|t-)TwOC0q`O$yYeF;j4uMg)^Na;(kAIai?c9t?sMFyo^K8huGuOHzKageZ+`T*eKy6W}GT! zmCHgm+q*OHn6ziJcX++ueP>1iQ1 zvJg28ygF0nDT-Kq=Rm9_LRoq-@!6BS%^%kqCT0IOhQJp;keSQ_$T+Wv2j?TVB32L* z1qnP)@e0)Y!K<@mJ*qErUX}PzzbIIY3M$LaxXIct|1u{(p`3_nHUp9P9Ej)P~t^8@u7zHYSU!# zvJp#$AmG*sUB7(1qRRnB=Pf2THwF_s%ne8MyYxe$6Y33J{;?#wHd+nn&Sb`_`b zJ3PWn6a9xI<$kh-#F|TfUXe~C*yjK=Akr8bA}N0O?u?;l*#BdppjQJUKz>0WIztt# z9;kz)3!LbfUh}p|#BLBC3=>Z8As_XzD0nf1W2N;JT*8kfrZf;@OKL%rgya(4yhdZ> z5m8FQz$6msa8A>)kFVp{N0nY&ZZY}WS#J~7REWSI9Opvdkl;@40H0>`p_*oXne^_S zl!Jn;mEo=m=$-E%7wk%8&N)oZ{+?K9LOUuz?WZAhuD|T^|L=i5@mv(94<)fv8-V^@ zoRyUWQhl!MQuGm~1IB9Ie{XYIr#oM8loYtn|NjV^b5 z3RG|ittucUcGbjwa|KA4&x9*HIen4iGw?Xz)3qN0Wnk~aP>YX(8Lo@9QGDp++nr%P z+RvDd%rzlRjOjJyXT`$A6>yb#oiFZFvD7l3U1pgXZ28*!+CIm-wLyekME3_$sMRQg zAs3z__d69DU(1B=$W@zAXp>ztV#-9fv<6Bmxz^t~b(V>9>54sTXXl4lxK&f;R<-i! zR12k29`O|(A8V?rc2kI?W1M!ej-j>ok7973IHFK{85i-om6*wh}nwE%|fPIxy92RgxwXFfoX-hmq^b_AEmeSn{d^^uW- zmynEbyhX&8{qlY#`%D$+14{;7m;>bpgh{Ox&xr0|TufBf=;~util@HwQ5^9pNz_9^ zPg`y?58_T}i}(lZ{Ynx28F{1Ml&h6c+LpJG&J#GHovfPoy@(^>0}+!(gcSzVJ z-uV{_`513}zLs`&E=5I_t3qG0D?StQNS8g7i+!{FdvT6usA64&)|4u18`(4n%GuVp=7Nh-B_ zDyAk&+E<)r(u6-oA@1LS^WQ*^GcN)cvMsk>IBjwpiaS-b8dAGUYOYn+g0W-Fwq9+b zPkPRCO~ey<78{|okwwQ?#W7WJZA(#Z5Mn=D)hrHf;vMr4yV3`b;xT8F=F4Xbqoq`X zd;roDKsf28N1z3OGbDtX#W*=Zg@r7hHxXX8n}b;A3DT_s#GLw`&i*fD6{+v+wCmtP zHtR&8@GZL%0S1CyE>J%%>%ny07P$%h_B|D40U~8}(aV89cQ%ID<1%?Nou;#Njok$Z z{6UEiYQf9zKiyYIP&ow^Nr*}nR07C`{K`J>^uF$N<=dC<%NEB2a5y7tVDd9jT~`*W z7gDAw6}^(=94{j}oFKSTE*Zk21+NjHu`ylEzTP58e?nU$K6CNGdz|p)M->+quNq}7 zxKcGF@8HPOb>1K@~6vM?@XB!q+5l7?JWGL#L4z7=9$ z+mJ`ZMO0PHQHP9bQdP)}TT&mJC7|SN7~*9BL^q=HIGjkdL&j z^dS~SpZS7J%R}O$Dj0Iklo&^@Ob!swf2(Uva{BZD5EjUbgM2v)nom?*8XJbffR$p2 zDnRu?y~y03J*4@>jh&u3XfxKNd5fgbTNN4>6bt_4RPSq_Rm@d;W=bBKjk+CFP{01qhsU(Gqldk4I%2;xwB+!+D*K9MfE5`xit~});Px(w~21wf5scBQ5_*QJ!i%-`= zt*`8fwNQHGKr`@AdI=~u3TYvAkpn}u3DDS2Pxw>_1*Hz86MUE$5Y@iKCE5rEG!yiZ z^*ODBAB#dJ(rfNhTH&w#*+j=@w%{a+>9hrBKMvp6Q;Xvt>TsgY6KSKeKDnsVja(~< z8V393+!+v)0LMcj*gxJUvQ25MXq<>l^@rj2CH&C&wh{Zi#16&mMxM{aNRo2}kOvDI z7mvRY;>g3-qTy?VWWU7B>McUym_(NVB#e^Iz{Rc%O0vZeLcO6AC!GKGO4*oyqlso- zdypj7EQk1fq{6hA-8bc$NnmZpjT8FG*U$ds#CI{hmhQ#!T&`uZ13eF=RKm4blE_N# zNk~!>II3fJWSC#;tOCLjGbCh7tD)|9SyBb z_`Cx^4W!^Ie}IqqHKx}REjv&^VE)1>_&|Jl2IVCAFvr*RJHKAMJq7zT>cvq=bidze zwmR-{tJXSfwQ2`Wr%^k0Tb)|Ff7F9*bON1&-oB>aPa%l{x=+RA8DG=)YXknkzOqS% zbpJheLPANO)nAcO5HYmR`$6|-?{}YmybI7Ns8rNX-rXq>TKyd$ zM7x98UFs}z3YtIR9RMck)=t4|q5jw@IszqK+X0F*F#a@e6_PbsxXE+VX2R{&=6JfHJ|Q-~Q&WvGGZ$ozkl7PtW=a>EDt%)f zMvBvro&WYqLN&&P1iA51NQXGkoA3E=@^C+tKKW|90WjXYw7X_*cUDT}I7JOs?#Eh! zCE8B07b_=#QLNuq(elX-3{1VYu>-4U439)Ei20R+6YRl|ue*>y;R0v`r_>D@D?)hG z2}MaSaRh~~8}bvk&R`;4JcW%w+AkRaSIyRu$VlHjZ8lm5Cs$IK6hh?}FJx|!7cUYQ z`?^psDiJ2#wv$dMUm$T3RTqE2zI!J7b*20zV_&qZhUiH{_av=j1P<}|nN&Cs)pvGY z3$Uj`K2n~f`d~ihF9xF8wZu_%{?C=1#h${_iD1$tCSoq&Zw2v z&jRFfkT^1?W#V?FwZCaYtd> zds>nu$9{QIrK@*YlnN$BrJV|TM5kU{nH%C>HP!30wI@D-QcmkQH>G9UH{+E?(8qn9 z`(gkESkOf@!CR(-A>w>BTJ0xeDe@tRv}q1zB2g&x5r&tX+5`o4r9De!nHaa~xBJL9@9AhKrkLN^4%IEHRQXoQrMWxOqR

z*sVhSGY>0kAx}>`V4FRD~&75cgNHw<4cBc_gD3-S&x#V(-z_?{cZ;_rShd z8L4D|tCUTthQ;^hN@ioW?V@pQ+^QetQ`s00E!(~4+Wg`i z2}%`ikm~IGEl^oYip3M}nBX80ri;80kT5%mU{wi&zWWaRDjXnHGBM*L<7U0p`TNdJ zoB{Rs*sn>~!8pftKa~nicx|eLc`^-IJ)l7nyQTs{b#4#b*D-j49EKHGKPb}3 z)p!2S|Neh>{yTBZ+xc(s)-T9TTNOb}x*4UTkiqAW0!S%wu#h`4Y|6g_=a=V(T01{8 zW+L<0)+h%kn?Id2-uh0q*HSH_sb{8mC|spF=eOvN&{tia%UoYDotE?p?25=(rFV(_ z+hT)DJjgx1j8nn)QDvmFGt0{2EJMx?5|YS@y+h_#DeZh6IKKqA8ddT0o}e!~4mE<>41 z*xXxnm(qEeC2Oc%La?R`#3Z3uD%pK3&it^I=*#iNvfSY*UcJ#fjO?}|XdjHKwx;ub>r_4g=f9@Y=*4Ow%b zQE$p?C-p|XE$!?d2*VB%W?|)3ak`_8&^tA7r3}b*BQlX97&^l^cOhqjC`Lvig46mJ$AL8VNn=Ka z@4i@PVzr1I(^R?YUY#*pzSkT}q;ZX>#ldfnxtybz(V~2gdWTD7{ zBp90j7UID`dUGby)K|GS7#^WI_|$G4G!G}@Uj46f91$;kC;m|BoNLFeDiz`{MugKPd4R_{K#2Z?kc9c+^bme_PE)aW70F6~14K$E0=kUi`N3QaP0wZj=sxA zS21HZN&eWCfTQa{%DN%)2z0wN62{SftA1fu0+1r^*>jt zW`iE2^yPm(SN@j0__2x_F}kPgX>Sc2wjlpzs|(Qc&Y8xH<~?)cKKWXF?vOz_1Q z^BQIz`cWMBbNxYX(V`Nq==CK2{j#H0Rt343Wfn?h;3M|dt(*cs@;rGZgu(D5As)kl z9$*FQlp35?o%TVyb8^@@IBvJZT@U-$N#830^pQtn=GCjY;y`DJKH?D~5i`Ch@9`P? z0Xo+MfeNzoD({PE+u~c)*eigZ_k&IqbPnWyjrvJy=H(OZqbg{&PO9Lb!5=v;c&5{+ zg62`H3V3Uc`pMCrrf)n`GjM68AI~BQl6kKTU^GDSt|u;y;H@5LdRdOJuWvLmYq5Y2 z*f2zNNIX|oI+^sudl+6LK|;PRvb#}b5MuY7&~z_{l~d3*8sl9!H2%KL^zx4XoogEN znQk?bzkP@h)!enE1VkQ#(N@mJ-(IrLMK&8J@b)&IYYdFP}awM zLMoN)#UF{!F7cnD$xh>c8TUnOcjg^&t@IT!{P!@JKz4p+;_vbPr3u7j$15N9&oVbi z{K5p<(Ex_m9`-Z5)W@Ee76$ER6*Qa2RnR;>;r!e@PO*HSe(5HeIQ%(Zjdy>TrdZbllfMqal@U-tGm zDI1NHa7}G`96xj^3NfPTM>NO(2>A&+4eI!sLVY2gDw$QGWbMI(B@u6LG(&AJRFybY z!vq&96Qy*YG@cEi&zBbRSx)X~kt@}p{Ggt^C)nL&lw*@o#NEk^BBMHA&d4KrcbUp2 z;=vDUyMp6^d{!+{E2z_J)+auCCX)JaHc5TlvXa_LYAdOyOKL8o;dsy`V-$+RbGfXR zH&lw!u39aa_^T7%=Eut9cvWbcUw*DYhv9J|*cTA!l)=2X1luF*UnL~AlHN-CQ&0&| z=pdg(K4wb`{Nr^=d>x4dRwMG8tr<_}O)K-Q%(pUsHkrRI$MUMoMD*AHe6D;tsCB;7 zU&Im}?1OFU>Ebx!VX(wWrwUF^_$%C z4jWHMkx=9m@U$g5BMf?0rdyc~%4B-0IipN(TbXWUx|QkEWx7b|wV=P=JW{?~dV`lQ zL1P&UK$^0fmD}$i)uk=Lj}Ut%Gd*e9U3waa9)|Rlmb3TC+U>zGCqsn0>O15QM0V+B zBXu1WJwA`{Ozg@9PKxq+7>Wgx{t9RLltAF}D?OM}iFTBcXeotwk^5RZ$K*qV2SYJe ztwtlbtz<;qVe&WqfNruw^3VVYP-JurdU)WU(T~Lry?iQj7gbWe0--;|E_yQx*j>rJ zf1!}%+Z(~#g6BurFSr`=@n07DP8*w}c@?|pckGHq)J`EDNPwq z+e}7r)e&)B{hsL2dv{=0U0mG%t=Nlw#^Y~F)PWfUtlV-EkMjF_phf_`eM`3_W6p%M zkwxAs^N8wZV^&#hEI>}AA0=o^t!aM!a)|r6#U2ayw`vfjetfY%yLtGc7f6>wE!aco z5QfMt*uvyS)yi%+U5yl+XcWoJyt&AZnUIF?z)$*q3vY2oeDxxcaydf|J|wjzaf+Ec zs5-P{o-W>^flv}J)5ak}1k|=jM z55~*Sa7jOZ*i0`?TYeFT+~wz7AcKNsDz3jnU?*6 z(&#r@m1_365u{AYr0;tDX0Or6KazxxnvdmH)Rth)bb}MYW@9a2Gv|(xN1GB<9&Hh* zEC`}gm@)?+Hw!BH7%iX#lL6({fx~TbQz(XlliVy!c(`Rg7qjEqwODO<1I@;K{u~P- z*Ekt|cZ5bHyz5334$vuQ|E|mV{}Djixx7kN4|AJ~!Q3YVWs96c!PWv=I^^^Aj3MpT zLCZC(F%L7y@;C!5vs%n%-Y_>=joEH3jW}}?JQ;GXNFkZu=?aafCVFgHmE==HkU(k{ z!BpJUrT9TYT&2p_RH8VJ8@hn%UGw6ZM}N)#uU4 z)5c|Tzs&AS3Gm9?vA}D)fEU=Eq87`f778Mry|*i-auZ6jMF7}?gDfYA)`h5wCsjnd z#Vb6-3_CCrGov=iTu6nAgUMZ&p#=jrDNbN{Q1|Nm*F_Lr&aKC~T4uXHGuTT783GeS zRU8uLaNu) z_qErr!OuURjz*{1Ar>iH)s!=aLNR~1h&DlpC<@2PxAKY<3VqDvs>$ChQAq%EBd09H z{viEb;jt5MJ5%?mc!E_Bft}%^85U9L;Ja+xZZXCY_sYAcC(n;YJrovC9tr3iQ};%c zG_n&{K5;)xnz(gO&v~Z--a&?Z=k7VrQ6mfBb4SFI$$Pch$#a>jO8?F^i6?dvbCZ~# ztJm8xS0{F^rq9)zZJ4XKAD;6RO_N(cJ`~{iVJ?Ygr>tPgKBiS7Gjh}ryDsuq1L1%9 z^`o5`eWN6KQg+r9qy4nny)eyAJDG9LuI?UU%E%yuqwc>W=rio0-A0{LbLP0JuHKYm zCAie>-K$sr`toxnt;^nm`~n`T>Nuwf4yMA^l)5=XT7h|!^UM)}6|2i%P3PH(6MSMP zc)KU~^|BLuVkdZ$6YL_&u&;?KYE+j`_M6!!d-jYjiOc>r&*Id0*ws#89$Kz}cYj>* zbY?>^p4fao?8f@jVs_h1@bJ0xjKCfO7ixY0N9R;jV*eOIpY~BGVvDn>Y&k9Rl$gn? zDqq3u!Yn$Jy}X$9Lst!-|8l$x5Hx>9ABAR>*UcG%bznC=O4Wt;88I z-|1OSjL%H{GJ`&4zy4x>wWso|K4tN9`9%MJO1&<+4RDCM%Dqe0j=zq2DD)9Slw$!Q z$Mb+-Ug<6c`;bt!2Wmjn+*^57nF2}9KR6);$>17wJ;W&To$xMTT?d6s(nQu%FP8o# zZe|l9K}PhyLLtL_>_CRV8~(k!G6B!=fFMN|4hA6_z-(!}0}K^EG6*iy;KIlhUI0CWf1V^WA5DDKEdKxR2!+B6JSghB zLHK6s>Hytlc1BZB>W*Y(ddl_5)2Zhx^%rJ4Q)qmam7Fng|JLooRK{WxX-vgw#ni{X z<3%nyD@@_7XXb<6HO$6Fk;ia>ZV3wRc0+wekz@3ab-`mYh7da z<=)t-upx`PI_Gi#I>t(QU|wtt>!&tr!LgI1L=xZX`Ebf z=_NW%lld*t_b<72TidaA#lK&6)WVSg4>2N-i@cDVT2s4k>pCvvrjmLsz|OVMN2!+; zNxJ&fWu7<<{zz5@_Wbly=nf@ZkTrrC-au^d()LH@FtOA2ke9R0GQtME0NGI z8X-`XadJf4`SzF9{DjbVhA6C!h>sZwu|L@FV-L}N#S=C0{e8ubwITFf5Bq~!fI@LR z_Z?LKgA#wzHyVvb{M%?WGXEYnjygwwX&xLM95$N=jl+Y#G+Ld5gT`M#V?|?} z_!ChE!@o3^{#JHykK{*jcDIZDAY#VxRP6UN zZ~dIECN0`(HSkji`_PA=R!a*?yIx1+7k}3bJJ&v~W91 zh6nx&oOvkB1{owLbvo=37`g>II`OZP;X+Rz;y}mG=>4}a{bXg}BPR4qCx^`geJzB+ z@M9dZr${Jm5s419f*Dc@t5eYI9JiVWhn>U2c3as)a<+;}0DY9&&(pzGg&8|bqSnjf z)=MMTXGJy{@_i%>dO~#w7`#S4=#vnL4NKU|qJXjx!4VMQEFlGq02N#dAW&PHc#mw_N03N{wiqm|{7cn0+n? zeUHQ2^Cs`obn<0E?qy-_WucVi(dlNGWhU)|Ih=wZ^-P$$Ir>~wdZxym0b1{~d(yj@ zRSJ7j9?YH#XK!Wi6ir=eNc)ta_6cF_Wr6LJL)%NN3hCW^vcJ%0AAm-^Q|15CXr%mW z_QZ7ngnxqk6@Z0(iF5-Xw$Z=*4=@c4`T+Cr@X48j=v<5e=2K|tX#FY#l-yF)$Tv`%lA6S^Ju3MP> zh+(#J2Yw*9Q<|i63K*K~@bgGv`oog`$>H?vCEsHM>c46U9Sd)Z(Je+V?6bw_+lbLO zt*t&@97eaN!{|w=S|12zmlr#=82s5_@C2*`kPTO)R&+?@F9%lU!76j(==*!X2b*=B z-+oM^Ys`;Djd?z6=a3C8rdcnh$rqI0I6knr=JDd1jPCJA5&{IBq1=*4Pbd`KD!`0_ z5ppr~6J=xRQD);h$0fkhu?0&OEG4AeX0TME_sQuit6&i@wJo}z=Yyx-PugECys|1I z3#~HEKPqT-9?LC2`rzPJx>9y-Z(~NM0FH=10MPT2YxQl?LdCiA=SNn}RJ}tBtyT-I z0Hokg6cP)uRtm9F+9ei=&TIWms1xHZ=Kke;-eeiIdQQx#M ztq7*Y`J+S)H-V=uIC~gyruHR~yk)E4j)iGY2Bwv1z}OIp9hmgZ6Pnb3Z=C{cDV)}_ zIBjh>&Em67;l;S=;XwrB8!R=Qf@OUiZPL}rOe@+q${j|IbcwXbM`*_ zd%(~h?SoW5=Iw@nV~ckr?nfSFmDSIA*l^DolvHzveeyp9(dVhCGphrdBDA)jstCwZ z@@99LLK1lB(E-x4F$&?p$QroJ@0>E-=SC+$?+lnxuL@F~T#@H5o^{bhC z2|Ea=$+FPOT0e7rezuru$M}RT>``?0+Ow^Tu=yP+E>$`IZS^h{Z+FlNV+UY zx-f`Y8l-5#iVB1jl?c%(EPmFRbI+6N8~P zvF~^hWhm5t*VNxNpSXxlE&U;WCfENsI6Ue!GW9Bq9@~R!#(n$qnIg+9r8oXvpatO&QRY@(Oy~-5Z$d=lOH`|Q% zF6r<6y*%DJ=-W5m@9)}2Cr554R}{2R$ycW0D^c*xREHfS&wVAz$fY~#qCSi~mc!&u zM41pr_umRnR3xRE$qh>>6C0@uMXJQ|I`Buv*KRU(;fu<^H*%erq&ify9sgONH7wQ{ zmTL@)^@YXSLVak;>7j~Mg~f`(WTZt(!eSNS6b0b~C0`}*q5<}kV^rm+-ZWCRmZ<kDt-nu-+^5e{(_W^npEAZq+a=!ZlMx{BtRj-w$Hdj2-=By+AOnqZ;W#1ERY}?kvnK%>Mwrx#p+nFR2b0&6fY?~AF z#jc??S9JGyYiQj$Ln~kPiV7e;SkAi-%^LqNsL(C-2f#K+a_(Kmu#v#4JNq0o z>&V*qXC*i2KFrx6x)aN66G)@`x zb~Mh7so-T!mIM_zXfH*hx_xnOsrUSPQuSENQu@$ylYbda0h;xn&DiZt8%bJ3toyT{cBv%%VDqv6vabu;wrrm`T>(qc_ zx3o5vXwF51+HBH3`%cvc&t^7i6hq&Me9vnBk{f6nu)Z}cvb_Fb#JHl1x$;V$VmqnZ zOLITpUv&fqaKWjQa9&Ct(!sHsiUs%$I!ccw{|ddvb63HmS2m{e;=gr|TdDipt(42L z!&)-(yYQWTO4bUzSqvsQ1l{+)#%B6;^YTRhCX+I13;@0$G1r@|40&dqk;8Kd$gq*NW$>}O zaY{KX^V=Bu5MwZhT4t9nGwhW(WRzuO??$iHe2dYssp0hjH1Murw23FrG$)7do@9y< zeY8@`CZoF4^o{&i8(iO>?2cbl-4WGT2X_=hqhIcCmM-)Z}g`bEJadi5N3? zzoxOO^sdmhw0KR7%#Wb0JzasZw{dV>I(%ICd5~bsmEU-$Y&czfE^l?sFh0dg&Z7BK zbJ~WNm)2^3(wt@HuSd@BSXURCX?2q6=ZZ`dcNduH-PG7{cwE~3tx1|qS&Q%TM{Em( zl6=u7x4JaJJA?P(mbd-tr~T@?{j&4Yxz;5r@(1c1m^fM9P*djNKP&q0rQH#Pn_hQc z?Z2;+tiaZqcMH_L1-C}Yo)o2gU?1*R$~Z|=-WYYz-Q>UOaDagJnV|id)c3$IprJ_v zGU{+&Xm|mU?2a6Q)|cJ`HM8q|93m9XtkUq6$g6=^?!nq*>?95?%t#}a*K+PFk~{pd ze{3IkO~ChtIB5<6na_-t^%)W$RX;F+MOpV5I;Ek%Fuv7CnET2hm+0_iC>+e|Ir1E& zn5i9>vk3UKwz^ACQ1l;#!?jSNj9T^sl+WtT`}JflJ4&mh8gwo-Ly@cK#~@%4iFSJ& zZuyLV4!9~Aw)mf%=5eIfXP*2cZWcb|g}{&1vNz<7)oLcHwWqJ$*i9yyHE@TT8YoH7 zbK9rT+4v1xxpMBzr8*=^ERc(Hy0YPjb7qgaw7Mzo(qhC6cH{Zfi^m%m))C(&aP3*w z(TC*pg2F_o!QiUKlsH~8IkSdw%*ZZr2Ip<~frMQ)k8|TfpIN|3>#k?)vy{E+;C`1P zbHx=3TykM-%;PtixSnZmXs^zTR9~z6#hXQ$zRtnCmL;QWCfUMSH@p9mNtw9b__9jF z+ZcgFXP1#B^EKDbzJ>%y*#lFyta+(01D3{sFE7}^;@Pa)cz{Mn@7vUo28%spH(=?T zU9QjR%JiWMz91?U8~duYs@Rr9ab&& z`ufG{D_7!Y@=tP*6qg3Fia-`P4R8NZncBqu?7{ckUJwJy@~OKW!@h0<$Kvj2s*%8; z5x~H{Czr3HBB*96Qv_xZb7wzD3LLoTdkI6e*)yt@sJ0Z-G%7dZ&y?FBBZRs3jV{O? zp2(bw8?<4<=l6E_x)CNNk!w+{W56D#=Q9Mxf}ZuuZ-GXiQU)BRxT^7OQM+huvD~`t zeBFP`NaUzLiOJgHN@yj~Q@)S76oB9xGJ-i>8}$+ZSrS!gU0i8Rx-&^lzmFo`M;d59 zmTw7#o%;vH6%r#I&;Bl}6{RSx=-NC4*U>iBNDsFE1r~dD188DCFxb9s`nS`bd2mz8 zcpfHbUU+SGMXTAYP*%5yN@13iyStySaF2$ZAn`J6GT_jsvBDrB^4oJc*Q zZW3JqL!#+xIGL-q)HXe?R989c{LwRLM_BKXP(O^?aiD-Kr>#ZQ=aXLXXmJdRX;8FMP}-s(*`J}-E`1j z{6jtL>mE%J%ZEzmiq{exz{!lt;au{=39Hc(GX}ca931@F$7m$-qsaJmC5>3pUm6)z z@7qtbnF}js5GpP-NgE+~{?q~d_aV?9*V*w!kevrg6=+24UH%&6MLg5|09BtywvJ$% zB2X@eZ+>jZdzTh_QsH%ZU{yZ(?FK%xXSKLf7{d_YbNq0f{s7AsNRrW)mu~3}ALCDE zPf{enhJA(+_C{=4vVbySN1sRfvK6-Dx8UnyyG$MRBsUmvVzl)#eQq1@@(w7?QLPn} zvN*?3pC@=f!1|ouE{85LKImQGleUM2)iJgW{_+!p{ctjLjoZP;b&OwEg3BN9GHkb} zr`@pkOJ3*J8(bx;ASyjXBaf_rd2^qYIK!s!U^MH2>$x6)lz5O6F^TTmmDoGhTHusE z{?fVKRJUHZYGWLRq0T0_t8UFlI8wimNq`$l)M86@*$5II2giS}{re{@Npj_tg<&PN;;5X$x{+&&$SQMhE$G;B8+q{lq zQ!=zcV!jY{v_Tqkb9rDmR5=3RE0{q7;QWfWtN%T}9FeS5 zMX)8ZbjrN}ynWrslxaw*nHzqxJ>{spK zxqKZ*hAiNp^g|uBB!o~x43U#vKbG=gxuUgt*BQO?n-kZG{eEDTUWCg0c#hueyFmFg z4g>snPFd}95WWcu+WnTd#zpvou{w%-9TMmgJvNVCUXJG|k#~gK-t{a0Ww!^GJUXN; zw6}CHkd%ms^ZO03`Z0J4^g$++vfC553vdYSZIObu5A6j!tJ+IfR>K>uG*+X5TCowh z(ER-EtOp>DV1uTQ_}|6o*C{pR_?KMCe~7&7rJwj`+AIJz-AE zqDP6kCocoE-Kx(_A89fpImLu6f-H&8L1v*-%fmSH(EC#T1y7B}LkQf}nQL}%tk{px%n7EI-}2txJvDY3|bz+ddq4+DX8T;59r zvl2mFI4(q{2cJPSO4_W zm)j&FOeoKgV4tg)5YkyEB0QGWL31=pg%%OaasL*UHg{7ipl{7+^`7w;vf%vBs*DH`NJ)SQ+1415fZ2l;2@?>+;)n)||Y zF04qQJE3iBjz~VT&qWNHVBHG`zAlRY>6MB4fM8F47Cgb8_}54$t3!PeC4wFXq>CiSIAgb7rr^I8F=Ek_4X(9x;qb-(~06l?hz)RxKm4@bsSKcYVZ0XC1+< z!bi!wLj@KjOr%BEEW$*bG*R9b@cmcUA-jImRpOT-K~mOJa-Rv{)4|J~t~mUTIYw*MGVixYrZkZm-@n7{l)YWS zy(0Nqx2RlkRFL6Vz)OAWju9n?ZcjyoPWSLlc8y-o+N zX*kXtDhS`F7a@tI|J`e3#^o22r*D~(D3lM}8zfp$pj6cN{mbAx#l2OyR)?e2&}jNp zD>NJJQ2_1w6o8g+dlhp6pf>=(Y~q+9qo~cWQfkh&QsU|b(Ehe+3A6ga<-szNHS2z) zQPeE^?92t7VBTXPkNZAbzH&L~-Un?yq=`6TiaKK&IkR~rFkqMr91$!L+efe~!0~q- z>x4C}G}^0ypuVW$v)X%i<50<+x99hd$dbyZ)V9B4b@g_i_Le51+StHeJC8Jn!f>!d zb|bKdo3;qSIM)GhWkT}hUnM@oe;%0pGr5WpVPKffo@=7SIFOaQLK2Z%HfkdX8JO+O z!MomHC&%DoX}p3tQ71{;9ATnFWBW5}X6zg7&IB4;H-gr7G=N-TI-ICie=UcYFrFjS z=yyF>p8j((%^K<77J8RWB?Q{n(0dYB4iyZhl!Scg5v&7KTP`$h4zu?eFaR2~`Q^XB zT`QM&rOHkI$t93oGoJz>=nhRN{6kP%g?z1`-s5y!^k1C)(|k|Fl2`N*8INJ?lnzQZIBhlrFOAFbvNdN`7W6G z-st^r^>1Tr@|0zfL7a28bS@vUv5s{UHUYMt-hwCtVs&$jIgYN5wMF6LAF_?TP|YRp zpQ1HaW0Kg}lK#;gr+4d>dHM(sjq}>$^C~Jw-wqT4>4nrjgd)wu?nyb_X{v5w0dFCS zkD#yk3@TsS`_)Cl1IRCmw3}M}D^Wm0@ug|!#rXLwNZ}EJ;KCLv(D&eftWxi(2_tCg zWI&liea1qcaB$m6EVOF^(j_ViH=cg|HN37aE3E#hS#V3IVUk+TLjM(cS$V(2EHjag zM!^5WI>=n)b6yUMpn)x_VA5)}ua2ejAU;08X!GM@BJ;d%__ONqop7JmJZ!(FmLRJ9 z@c&Z0Cc^MFh<1~&VUP7uMKNA#cvHqcfUV{eqf*|q)z4*Np(Gm@92EuL3l$+>j4zH; zTB#DGl*Gn0DUDhCT=hPYP$5@8)oY}{E>|*;-GY*7wj~lI`@{}EDx!RoIwSLxMK(dQ zw0-WMUNM%kfVjvsOyTcY;BhcE*6R3udICSDBHD^1SC+XaC)}hRU8GCtP%4j9t8*q*cHe9^&pZE9(BeN6C?(#wf z=PT@li4-I=9phCUh5I=8moZ79uvAA$V16hmfsSJi% zi1OpMFYHFQdHcGxYldaZoiU;buc8^c572;8C}NJ(50g@#2S^f-qB^9&k`^lW8>wA@ z+6r<~C0v;YvV%Evkt(q5=T#<)s+ zr3}#CRvC6BD1|7=5bo!{^`Dr595EMqnTb$6>Z6=HuVMf)Efc6Tf8; zjfAJ*Qfm=gz}5cyE7Ny83C_SqKI6q}VsD|6$fKfn6vC#pIv0cxV6id~h|rswK85uL zi3(4sl_ARn8sL5>;+z!2c|A!K;WBq5rBaog9IMBVbSN z!()gcK%{+NKnOVQD~cSNg^ZVkt~2@n14JP87S9p1_#Mc?O=B;qdPso}HzD!UZv%QB zs_R9q(j^A8K*x=JOlZohAyy_5Z%MDg3LX+fX&VyeXw?Kp*|~U~R8y64Q#uPz&-76E zVA3#)vxM&)-_b_8Vux2m0zYPsWToNG=4*upX5HDkKH~$47D3FcV=TBq&mF$TajkVg zhx3cz9>}@IVNw!&xbr?E&UMmQORu+%)!#hsD6{rq*RlU_4eF`Xf$kyc#Os?roCkM(KAqN=Kku3Vo0<&tjEJ#;CM8m z;Y;tzg%*YV17hJCd`VUFz=PxPMH4a`Q& z&RSCFlVF~EN$TPqn@Ahkq&nwUEs-F(elOV-<@1Yt2PZ6X-+kw!_wma*Hj;EKl};Ir z3LoUfi37?TJbw#gJ`7_YbZtB5pB$aG$?VLk?mQo?iG7zl3C=Apt}0tQrt(jX4w)AS z-V^DADbW)CoQB)NF&GNyTl@uQ+Q;xxYO%HaZz}1kyis`e58?UvBB#vz5tOOy-{WT+ zo>9nM{g;qGK7cJYYRX6a{O72w`iWbgfUj!34mxpD&r9FI6}-QcA1vkWRtSu+*lZXU zvu)G<;aB$HEg4X!FzpCreMK~6x~sOJHywp5HB7@_ML z5Z#;!Lg(!nu64op8cV}gqTK+LOG{I+XPl&s*p%w6><$8?3Uv+%{!h^eYuA619f4i$1V0AVzdI7c_$gh{IgK)QR4lsRp>6E9& z`Dm%3KbBY}PEfU-c`;k#{Qa1^^|8A*IBWqc|4b+1jVE$q=Zdx)1n~85SB3tN0@7se zFE~wS&`=}J8q3oA`7?QfQw5iMDS zSW6XMDrP&GY}CsQ)gnjNYS|*m%BpRK+F^KNkG;Yje2P%gLRr;4?@?wUMR{{F5o!BR zyC6ou6lQXBk|tL%9E6>qU5wGOuKMK%81LYUHPp^v?IogtbqxKzIM zEu@fr(!hc>bzL5(4B165d#jI(_z@1tAnVC|XXLv|>vg?+;E6n`QueX_wpE|}8K!#3 zq)Za?9o>kqAB|Pj?WX5;l@tXGv-z0PwIDOn>niB{s1`yV97>4N2`Yyi3E!EQ{TEo9 z{^04+D%$r?b74niIwyZw#gXlFu<)eh>xs*x`k=haD^ylO$*y)W$u}7XAc4vZw!wO3 zXv;34fOIlq6zU3EFkJzoxHFWXfs}wc`xa(oqz$Rts5RD0#^wcTGIvEZ`agdhibkQ3 z6`&R_!ul3_d4Iko+MCOHr`7QFUhM*Jd|)f2ZN0`&XjWCQ)Fs{5j`Tkk0~6RqrSm`! z3Zj6K7V172MuNzp3uAn-{0+xG|Ko0p1Wxo8)+OWkI*|&B5`!Yh|CP=;$X>Lx$;}hr8 zY7pd|I84RVvMOUw_I;kNp>c~6^R3SV62-2Gl=6U*`lJvobs#019i(uBF>pO zY7w3u!Dw<0cljQ>m69SK;`X?jTHNCa+-h2|$6S!e$X|NmRN;NUUi-@F_lGz~=dnjq zmLbmd5xbIobWV}gj76V#A6dSO_!XN4LC>?a{Nn;yC;S5b7`{@tOY|7%G|Ms7{j#-G zcVbC!NeXLt8Rol9x%{lT>;sD3jXzCzrbyc?0|EYX@2O~967~k?UAVspE7Td_jXvcU z+T@nKmVX>1#VJ>RbBZ^MIA1BcT7PG6`h0u%`RLaj#3cDd$Tv7rd-`|ChD^ck-{Za}@ENSYIo_488?6hk^Bo+jAk<$Koz4;U3ir z`=BkY00hluP{EnF3ir2kMx%Zq|JH9Sw&L8};L@An>MuvmAd17Ck{q=E9#ApSh+-9#HSTtPL`PFXa3w@8-Czf+}7Za92k;5yGvW|PzQdf$ZI$zJyu@WNH!y)AC zR$8hQNCXpr!Jo4e{7_uJkV&Ex(5Yhyo{8z5JR|L;{=}S0+~JBgRxf3;)tQDC$Q^*S zmp#^VHtmc4%HLTiO|$r;^9C9pbf(<_#=MJX;6~`qRbj*;^o!@Cu8fmUGa(^f^k*wU z({nFJvKssY7XKz@(nvghYP|xm!e+;e1@7#`jytw|x|`_;&`BZEi!|DCy8fB&2Sk!d zph{l0^)%D94MhgY+>aEG@wwuWP<9zI66vc(r%B4|MfNZ%Hu9(df&7Y-1ki~%LD9sB zm`w6H%XDSi;%L8ul#l0X#3U&vP00X)1Iul%R9?kBt(Nmfj%BQH{O>`K0x7_3Qfwyf zNx{J_$b?&|o3>S$7u#+ToJ1Dk?%0pMijkkp*|`p6wJlHshP{H2K5#e#x=0<^AB}jA zyv8e4GOp-faRs)jp*mONm2U_<#g{RDcw5zrv1hyQ*sVeK;1fqxD~>U@ECQX>we1YB_#n4;4NsI{Z3mdzZ#{pHA>$(2xP$B!){*)r{f zi{oox9q%tYVYQ9=Mj48h&<;u{n5m2OAbdzkp_jHx72o|sch!qo5%ER4* zrKkojiS5<4QRa&0Iqwa66nY29^?QZAIvuW?)*RYcU-3jtTd1}|PAeJiX2HL3 z99Yk<*=X6hKzy8!d6KG{s8c!96cvgud*B=?uci9#zb-bnOyt+y?tI6jHN5id0vO=p zvik9A6TC(VW6Nc~vD+wx$<^I1d^05#QA>~Gx7P<`KzIs>dcM?}=UIVj_g&{>G&=&;rT}?V;N~|~3P=EN%&|;6*g#0IOm)IgXPU1Tk+?r@6Q|R}^ z?{Kg9QUXrAXbmRQ43Xkolit}PiO8yyjSF zq%U#P`tSj=@1A?db(zs1K_2_{*DKUpv%Q{Wd=OJ$;NSx=Kh|YHGDdH6SS;JdI4I44 zQXG_kX2Jd&vJ4`!HMTUOtLLaT@a?heG&nD}2Rc1Ods`G^LYHgFW_9HkWrM)@I!E1W zceSk}WyjtG&YC?#V;lL-Lk}fZVNEOqS7+h_G#_l7m>&+tjF^c7ls}K~&XFv(^W4*| z>yX6PpH_0zs*NDyHgiY;`{%?q=hcdC1q37aGuurOdQm0Gi&Mj@cZ5J`?VZa2%2#J` zSk=Z`-ku~TqoTG$KSkJ+1DIOVdrg5`xz%(2fq<@H6R{EUxl?b0_)qy}K~(fDek$~& zs~Ca6#eu*-Yq?h2LW29f22qlOMvwzNILI?&S3&~E{Z>hZ0sGY^*Xb1}ypftmtqGP%vA7n!qAM?KO!Q-wcU0PTqF4=I{v|N1Z06}s$0m}*UJg^t8Exo( zF?uwfsAHJH3zssiIcE2W(Rlv-tsh^41-R!dLRy2czhyLvWGMovwpIO&Clttz1{V#6 zKEd=6PPM6wlY=B6aNt)aX>j29ImxzB>jKw@KFP#WJusU@wYK0ycgqL`UBWId4Xw4p zvcTI{NcY_k<*c#AL+#iI7}=%?qgG`{UOMcZnKVNzhVg5Ba(0OJa^}gl$<8lKf$->m zt%Fr6uPp^Uj(O;e%c>mvW8hp$x*Krx`eMEn_4h8zCFLJT9dZ-e-^N{QVqoK)4rsyS zUznKqThN?)o5)Qga@~(b2nsyTmIjIoV7*3>cc+{=XGc?$#!|SJ;0Ee5e_9>NCZ+5J z>PkgMuD&K-D8BU$gZKv%<3T!B?}-I7I7@+kH3STVOJo`k*U&eV%Ee5OC#Dv{| zp^wS>h^dy^+V!%W2Qo~;G_&L}vxIRS8^&oa7+@9izNMU*z2=UE!rVUohBe(3XlAR! zvnAf_E#wF|S1LTQ$nB9JBNIS9=jM~}Lq_93lcc+T-u+e6);ubVs_|P0a*~I7wC@z? zj_D|6vVS;~Orp#FB~7_Wa|lDEv;N7+$^N~r878RDKF6dSwYuPHWu;b3!%h$Mx$^p;gZ1#2E4#-8&?XqGnlDj~?N=|- z6jF=bc_-#S_;=|FRkNq|C~AxoHEp9*jcS>n+a(_*M*q%wG?X9FWpkbs^`>=%6V+>h zmT%S*sqGAt_=(hgnU?PqlNh)konWEp!sr^Y$Cn>|B(zNzEdWl@<4rEZ7XMrDvwr)1 zB;ZZPs?_h(%&D{+bf78t@RCT#52}Z}jOW|{eb*G(;;feOm;U?6D`1?DDM;$5%0qoy zyqvOIvF!VKrKC`<;VbfMbwbbNzT0q?#~c?TPdc?Q;B#R^lIHW}rSuT>@CyHtP5e(> zt>>`$JKA(cC(mn?U% zya)p}TRbVnpBX*t2(CBcnhy`@cp*znl}Pr9i023+MSHxi|5g=C^L5D}5k zjEmCaXhL9kzVjmU=%fzd*eaw2P#ldRP{Ynsv(VjFeBl6(8GkFMLRi};{C1k#vP~-> z7ULM;<(8W2sIgnG_oWc!y{2zs0Geef8hnm|23)8+I(n6fy4Yr?Ov=4Z zRvPrcbxe?*rl|~c8h!W`10ZlN{A$=6R0*ShV7b}nv7-UDEP~u>+uH=e$-IpThLW?b z3=ZqRVs%(D<0hRVPMKstP1UA24Grg#tLAUbo^E8?jnfnSbeCgSt;q&*UuzHb?q#qF^0qBlX&z7@D0}Uy10lp$m9mwfuR3Pi^p_zk zbD^g2G4BcbG_vC8rabzakK(^%(6ilW2z&p>h!HG$f*Hn3Vh;>Y(QUrq6Y0zyJW~iR z#_&ozJ{@LCW4f`jTS-h|6(yi4&gQx_Q0SIaEm$q*IkD$85fpHb@+p8pg^2&2Y1z=s zLGqeNYS0y<6`8E;dXGRK@F%E&Kdp1#y6Z`jZ%sDwA-Fj5VMG1DHW+o{H>~6Md0hX=AGQN9X-ge4j*iFJEmACSq z1hvAT*h86g(0Przz2B>5)J@Qe|Eh%qn@W*%cDzxnt?~8!Ca*POtK2(Sk%t7yTG=9Xqk<>l-_S)Lctbu!k zmLfR)-f|_BiG6%2r9u9q!yN0B`L}@PnNht}koC#nnWnjiHfg(48`EzIDxWi>9A3~a zWF~7HK=WKBa^5u@$f@|z?R@&dbFKCD2G})Bmktih2KI=R1hj3|OxEGu7ghU0ys}RQ zS@HZSs|Ry&$&RYRoKBT9^6ze`I(4(6EK|U+_!AMP08z|TCF$S( zop#jDO;4Js%17)vtF5~`3r-VD|JJGqHA@~l9t}*xC9l)Z?eZz#A+MKDNYk4`96nl+ z@n=h4JH!v#wa&!B^m)#%%RX<38c+ZI7GrUz{K_-FP@hUhxHHmu@97@w-33de6N7p~ zLlfbvsCO5CO5NRFeTv&lpNae5-!N(cl1{mwE+1#=0GesMMeHaR-0Folp(;<2AaBSC zx6h~=VgEL3andlsE(Q8BLcB?D+eVt6_wG&s&KHY|iIet&1yq&O+0l<@#Wa)@sXJ|8V;B2q4+*zPo{qh`q6#9u2x0b% zfQG?;f*wWx%|wX7>)`$io<0DbyYcyUTws8XKBq4g|F^V+yCBA*Xd4M#VNBg=By1yV z&ZJh<#=>nyI&S77v#gNPB7=}j8&^BB9ibd)kU=X=zUm!C1- zufK=3tyKONr^%O4KmLo#OYxKdE(sWOxLcOyDF{slI}8V|wncz$ZN>t1q7?NApN=<^ z6J7X-ibv8aE7t_OT$e2!39yWLcTS>jG_aW3Ix#A}B=W0|v6(o|!FeRC#&6o}5Ysa6 zv-a7wi~MX$c)Gh%sjaeqTsS1h?=+;LCk#Y6vnn+wxx7he!!03d=eXdEMdm;;qF9S# z*;SBhV6ST5MEb|*0{mZgm$q+RHl;|cS{FDfKssp-Dh%4xY#)ss4B%J z5H0*aN2@2D6UH4?<_gRDH}v?q`YhY2Vt%d4=SfTT4uEWvR{8M4UxR%N2IJoyl-Anl zJYnKcIUz;gsy=>*U@6sx&|NypHA-T=&bgxxg|1t5$w~J`kGm7b%kY`*a3N@07Y0d> z@t*92J@UM0Ns3pX3v+#%8otlpDJdjB9JF5@Q58cc{Ve_tX23{yKu|Ye?&lrd?WeLB zB4jb;NEm^4w7WBBhbSRf)~yW@RWM~X&j!&&I?q9E60f+hC(8-iKVf7N%BvsK%2VKm z7S7|Zycr2~)ptXw31atH=bSG4Im{i${yQ91!-u$c;C7{eKGq2mPkX4FpRo*ED681e z6ZNuk7bsAM4H{5;|DgrncDX@3;8y+o@JILN$Bg56pL5@HX8RHal?-2dgk*PYZZNkt zT!i6@0NOpnp4D;S*keI@gTRAeFUI5f=jWxuNt4Fp>BmQxZr4_m&&7F7g^{OcO@R@v zIMiq%4;C%oxf%Vlm&C=7B64p|#eXq$I9`uE@K#O#YSJD(|ttes9YH6^I6_D?-*$7cm(8IZXejrv%4f^i(4&`=PbV;5+?PyUS}N{Lo`K(YExTfD3-*N>GwJz|R)<(%CZO=AcIC-X`}NpXJoH+1L2~~2#WKRnI`1%OpZ@6i z2PeC7b$Ji0fdvcVY)zGRmH4$mY zI3S^^F?l=XH&|bq6t8h0RtR3hBoIfraA*v z}j!ln${>9^RNYUEN!4_x<+zXY{Y4hfeaEYM@&{a z$o%_4*nd5`VeA)9jW6WHt@4(81^;YZl}wzwwa}l4>_!vb_TuSUaop2whRESTQuR@s zMZ_7QD%+b}w5p%J{D;)4dB3>A>bwv|N zs%!-Y96y-9=FJ^1x9MCITEC*sBAbXT2KIg{fAFuqvh%O)xMRNm*h*s#EB|w3>u+@S zrqlfXRCA6xP^bBzrT&mh`y{gH$;T}z9;l!JoUP@OV5Kn2LG0b6Jw0+ODlw0G!B+=l z?wSZXWq+%>OdS@}vt%!#aECdSOZ!LfmPLjRubNRz-SySQFCvexLKMPsIu%158B^a= z^WQ25N)aPfq)M7BphFtX+qG)Hpi%`uSB!l%g!#^}yje`3!#AxV=Gwc8m&(?0&G%+z zV9ajuI%2ug6$WEf@X@{B^U)`)!1=(~(4c7K_*0yxDu##bnEJ$OpfE=*Nwe~gTQ4l> z`g3$%XgUWA9PbZBHt8lyz|R*_;8!Sbu}K_YdZ3f{IIuu0`Vw)M zHQm&7!73vX5SAfcTD zzhJ;rLF{|4PHId81nT$Xkbq|wopK+YzO77)ik4cF3ZZYAr%Jfmr&`f}dwvAqA%B#l zz8DbhRcbFNb-Tx|Ir>?dNw7Luxee-MNm8A|W7P3rjZaQ3gYxqNZArQU zU&`BkA<;K)JEui#6D_`m0_gDgXL z>%_v8bfWnl17G>rt_QM+duDp?#W{6#gbC!sSWYCG9aK9;i+Ix6-FG@TBNq-M!$r!M zKi{vBX(SQ()687>i$dLAkq0#W@6r9`ZNx#!tuMNY}Mk4)bdp9SW{jCj^sS9qVb;Q(dr@mj~Kr^22dTKDcrDb6zW$;`#iBmC>{}(;Sm? z5)6Ma>*@s_I+q%j!)-KMNUkuZJHxxH(UEmYi%uqk*j*D0mxD@RIDp>@zJ=u(V9)WF zqbtN){}$tD+&_wzp=VP_ObMEG{U^sthj($QgP~BhJ5k7F^Sr-nZ=%iaj2tciSAgSJgPT)L)B@K-8wY zoWtVEE%K(;^hBx^8{BA7tUvQ9HuQEE3wfH>dc0UXUW062Tl>7p^McwcPmrV~LWSWd z+XUy{N0I)uhOhhs($*^I#n2mcjO_>RDaEMu2GPxC1m=oX3s8N>SSy2)?TX%zBtyR3x|G1yjQJ%sgpyy9nC5 zTxRZpX-%DJ0CdQ?J*dd1ZYe>u&MMcdZsV$S^#!m>7;T_nZ^Jt(Z(q=qNerf_3IPy< zc2MD;Afu^=c2-}ZNnjT~b0n`l^8bUIa_Y)ET>-7@;CNceg}Q7%Pi2YFQ{q~)WCQW zdzU!_YIxfpn+}Ku6}vjuY2Ec~43{1TdLO^DGz~PA_>~j?24)Q0)_gNs>c6>l_}RW; zgpqz$-|VNx$n^B<_kk`yBM{?i+@D&v4Fw=O_4n4mL1sao{wYAqw1X?>QthvlLY^w2 zBB%Gb?x>n0g!B^8Hz1>CIj^d1nW`A}M^}kNdHuM7S(@Hoi8gkU)FvV?A1`rQ&k_5E zJTrOJ-|hR;ZL&+^inqO^(n|w^x3epfOX4iI&L`rRc|zs}VdR`5(hi8rP~=V2`g;AG zv~U&3J8u?nwedGQ3DUwOztH{G{ktARoTk9KQN8K}HFZfDs_P4Ul+F6GD*gjTpsX+44aqPEMc#gWbRt-v)r%ozVZL(d+mf)u@|_vC7--0+kav z?9ZH?)ylxn2nWMpSkvZ|(Zl+nLXF)arhB}4yiajJkR@**mUD<2#^Y_! zvD7X0%VmOes9?%B||6F$s`GzcBp^OfSlQT=~xq8 zWMgidneJT=Ps>+pc*Deie9il@PH|3mW3Ej1F~-TqSO}pXH^FWyB02q@=>mVj9`7k& z-ME8`@+uf#C&hHZa^E2LeV|8k}+vKyeEU{Xjw`1T`j%wyt>cnkWNf|JOM9vL} zTULD@z`Mji99t8(KO?7l`R${t5>&PS-6(edJ;0}Zu(_eGQfu2lM?q$@0<9zSAL&XI z3&XH&)|b$^gW7!Of@1cXO{i^i`kIw1sy~+;v9CdGc}oNZDGW!uS>O~C$_3wqAD)>n zveA@3Sl2`O#p{9xve#lL!Szxq7w7heCfL;jk3%=KHH0}Yx4*O%=T?qWp`H+0=SrKm zk+Y$~lv-%#6~zAH0zW6`jDyNuFR6f{I8HG@+ho!pyiJnOi(c3#JQpMtzOKlsiMd-E|<7*avjJ#Jf?)4`1rf+D+QGH%`#9gP-kZ zZ5F6y=-T~b zPy9~OCH`I)_&sxZv35Gk)odw-8>cpObY(Sbm!^l2LDw<2>yptY?(XO!ZxCYV0=Nj#PYbJeo|oYJEz<- zacEJ0{S0UOlwC;857M>W@)=IPe|C;FzFhyPU1~V_HhKhfYwSE5C}MT{$-B8!UWnaY zqJP;*veXI>HnodZjI}!Y*Bwn%XN8`cuW*zg;qX}Z>ZJEPYpaU`Iy@x z)c796x1lPx&{&ko;M-0G*yfOtV4y;miN5gOF_?ARucx~g9)>rHvrPLsCn`5j&aZwy_n@r7!v1fa zxIPz7=UW#?Oc$)-eo-Wr1qyN#5Ykza-;5rG-K1VeEO3etkaJ=yM~!6+CGm*z$mAVG zglc*H%ub0i7Rggm;@bCi!(&(M%qZRpviSK-G&wp98mKP#=E-DFe*-)P7SEK1N<}|mDJRwDcCGfIW7VdMqOMzbhQ<2zSoYZiLq_Hh zEEtnO7qm(!vbTX&PF5|yK`l2w%Sbzr!ys~YqmVzqN^QjV8?@G44Hi=ET6Hp^!a^d9 z4epO?XH-Ujyw~f7gdfoE@0$tlcPa;!@@^{Pd*^^VbtRb>9y0!7NZEeoe?6L|08JU( z^3L3s^KZJfr?^dy*XA(INGxN6bUWv+khJ$#B?d>iFp9t4?bffJee(h=O3^XGQ@3~O zQ`>Yx>i$rn5f-?K&#KT!O6Bw+@ykta)&A+ID-DL3*%yA1SfU%9{j z6H((cWa}Uu`aUp@nA3KAX{ z$1n-oTF(XDl)GYx*MKY8$O%NwB=K&Dj)^a zmo&uw_O@}Y>-D*g)zK;-*8>fSb=a#VKK~LHRtaX`rZg?R>850_>*RbQqMRo|o~?(A zU?A$@=U)qZf@EgAp=jy$HD%4g*1|I?p|B%r@?HZz>_LJXJZiF&#fc~>R3T6%| z|2`~f2Kh0{L&1Yr$o^e~Lo#8ZAsAwvE=>*oFH(DfN>~Uw{Pey(dKg*7-@jW%B0btj zF(Gs**gjNAzfG;<=8eV0%^-KYO!q74TC_&--4w+zC>YJBpiJ4Qi1)-7-~1UTe$T^H z%D3!zA2TCstqf#volHGUcbzPkT-%7Iz~g98I}6 z&>^aXNtEVjj&&jYVC<prB!lxavQWpj1%aAV}gW{!m&w<%z4XXF+laby{ z-IZR!wndM%Az`#j0yHEJ!^a-nD@pOB8Ei(5m8Ui4;4jb$esSy_IqgrGZP$7n4F5rS z4$)8II<$*ivXK|!EK+RqkemFOb^W2_k>7S3N4DACQo%)!Yrtu8*G#cGhLp?`#`7nF z&YV0voZl1vnbMi*}Szx7^rr9Auyk5cN zB+6M}tZ>AwcJ%VjxS*q?g#lLnVJm5`dsZyo&H4WTaX^m0l^KsN)b#|UXhZ@#x>U$n z4Sk)kB$I-|?es^&cCkRL|4$O6zRk%I^l;!6u`HFXQITaZCGF0;l>WZ*z553G>WLxi zAV`XE#Nwf#AyH}s)UN6m2)S%nNm4a`?drQ2(O^_8$DkP5D+v>6{=}|L)a6m3EeQ_T6TI>1| z3vkr^@O6GkyP{5t8t9D%XoNSi`(z0@zAHgQ7&aUPLR1ZY07+C^h zT$QBq955absOB-qv0~ql*x|Qn6Q{{%Falf!@~YV&Z?B5F+L**XX%J>`UeS(lr){tSXRBO>QoGGmX2gn zWsO6L&M>QybJPYZz3`T#BNiHiVFP_2I25`uFpUkMm!^nBe|BTlz5-_4Ad7=kgSe0+ zoCE+>X>3-XR(8lJJA@MoT|UMWITJV;vvc(ZMuRMsN=AcnT~eQ5PWXwe+K<#&J2Pfq zMfxM0j=HGXI>)@lv$!Qlz)31vMVGW5Rz0dM=!oXx-cgy!%3Bn6m;G}L)mhL(S89%# z$uM~q4uSF(HC4^>sICQboNxB!=u-_sP}T*EG!?O%8`Q+D?XX0 zIRPZv0OGvDAF@bW{tY#MR6r?;=$L{+=K2>q%p1sA2MVrB&<1e$8j2&F92_csY0SB?h*8q;)QiUi-K8d^>gCZq*AvD*nb;&uis#lxvRg#3BH#$z0Z zUG!VC^l}qAB4Eix$^r3ue#v7hEiMvz5$$u#Ctg1jlRoS$=qkYWOs8QD7GqA(IF%zK ztN@_6nIa_Wv;!t&)!agl zSSW>&HU`s7Uje360Y*W@ve2vvDZM$lkQGZfIfu-TSm>QF?`Nq$v`<(AwSJbOn58Dg z5erpAer6ZsobYX9@nq7ZjV2Pmmnv5B0u?5< zUV31Xb9Y8NYZ;3|X5fsILYocT%_iSiO&Idh`3P-U6AZY;INqJMa z*k}UZZah?;UZQfAS{@1ma3uAgpi<#W5u_rGlyAy9h9#!s^2cam#2o}6Rum^&k#DnA zI>hz*faG1&ZZyDtTVS@QBhIqnNSp5^WuUF^e41E^4Ov^;{b~DYyMfSB!b>Ns5MQu( zK=Wo2QAz_bGG?;7!?>gbTR~)**36Jt=5dw4f@jU&>$s2m{8@4T*Y4f5dLt4ggbP1S z7HtJD*#F%-*x4`b|L(Utd-eYBZG1M64!LbKzI^eF0npS}p=mVG<9hG0iM-28Na#3f z)y}2~>Zshaw67(c#_0ex|14U~D`i09aO&^I2KrS+lB44+0R-Oo@}>2-VScq9Mk%s! z80Jn>*bkLT2!c=)M%ix2Bz+RG3;nYrKcx`1F|a5wu1^t+NSH-tJAw(~=Gb`LlGSuO z7@n#yNhf#Jzn(U(-7pzx>VfCKtKXGBkimciX%{u$v145U`Lpp#bW!uwzu>fHp|@sK z!+5p}W2|-?U$Ai^@pBAdqryqUurqN$pY=n}m}jn*36+f0#Gy2fJ;Jy3bKagVa8PebqdHd~T*`)t=<+ zzOq^P&d&P}v-{2MG^k@wmOUnX2wQ}3g59lnp1%&sYKD|+egu0L605NI4MeT0+UqE=F8cDN zSPHtlbhlUKV`J~zE!wZ>Ul~gwsj)4(D4)#B)W~dPG)sfE$&I_Dd7>4sam))uD7Y_7 zh4OtFX(Sch61D(^DbsuhQi?upSfb6T_eB2UbTM07Rmx!MMU}=8{LR^E&G?y-ciz*R zSAQ1%pM@{ZgYKWhCfd|zucrDECRP*qTcsrBsjCvCRjotC$yAa&YM*fC3{$e%I3ucTahLZ?pE1vue+t z3J)g-6PPm9tSGj`C0f^}`#`A+uy8=o>%plOHmn zc5q$HQ|;QduLzPlo4&JOEN>6zIZW^D%+94Ety}rozU7%Y&V9>iWu?fR)w_!Xt&f~e zSB}yT1m+k-Wp4Z_X^pH_wi&ttzK~GB2f_zE9rBsu%*7%yt}5z> zb(i&nw%A^Wx@Rm-U%KDiY84GkoA-1Dr^{_(*a>c;CMQV*pkbM}lv~d%?M~n;ZM2=3yv*}`IkEWddGsbh_ zXc;tK-Do%GRYt7v-gU;TkWuxhDSfb94l9gYp<}K#c5czrqelm@^|49DNjj;>rlMDT z)SODzM4J`wRu)%QD_&IrDqgawECbOjD^HV^Rcg+UnA-9*i4-Px>EzKa@@XKLEB zWX@IXb60&)F;~Ss`K1`4xq7CQwt20BDuR>N+8PxBv}}7+>gH@*a_y=W;)crAD$c}0 z(s{G7oKVqaT%%g@{S}IH!kFyTRU&z2KQ_Vu&R&VTPe*)Q+^ zJvr#q{2#aSf%#~C6La=X{5jrUr2M0E4sU?xBtJ_HbEUYZ&Yr@)&v~bDM&q!HUMOt( zfkmV-CMgbain|Sj)N$vfU495D?+v^%+gOjR6Xq+5X)ISa+LP~dh(DXZ<(Y`YZK#KE zLVw~cOGFp__9$Ty4JMC%FNS-_-$N7X3P~stkm#K7zUV?Q?Qf4VK?-le)`vRpvx3*N zj0JEUF7WVN!Ce9^k6CP@Tt0qyBiq?*=~%F))41Q7Ly|UspS5|QrbjUb(D>1m>|Fkh zDk1E7-ZiG<-)J1tczCC?l11bL8OV~&T$|I1213(oYUMhM3=Veb*{3or&IZ`2Te&9? za@ZTn-7&RcQ#E{ckV@6L)&Q+qsp}4@Y}Ab9u-xLC(Ie7U&_ocGhbM}gIV<X0S*I#J4A$p3{g7 z-SA5%8rFtd%HLiey(lI+di^s0foJid_$Ub1fr?o~y6Djd`Ridkc~oXWn0kIqb!QGV zltx#Vx!~o}nG7xJrcNe&S?=|(M)Bo+eo&f@2#G97B5)ewNaGK;U*2XSg}!;DeuWra zbEL;ZV;Xl++aBc^L@8sGJJ>4)ZIgS!;gs6;Dv{nrm* z5VC8_FlcsBQ-=y!ExS_Jxu+j0RF3nKJ;PCmtW)a694crifu07VH&<=yfy&bKr_RzD zDEYZ^M?SUcR(F;OA*|>ybC%1zQaIf;)84diRTNt4Hn4DtXEL1jR0iNqwPi-#-E8OE z$Ay$%Q918IWecZ8X4JJ1v%G9|-7xC8wRICNsB9Iml%Fsb3PW3~_eaV_`YTOGKdV=C z!lD85ZguN*YG!Ge2q$cgvM5qEnAw?8uaq7-xXDVWRkEAv7JGhoD(9-hh_*(hrq4VD z;Df}U8aK$p&(9N~%ggS}WGNoz_03NZ*&p(llk)pIsr$Bei1}_buA31lCdYi}9A@{B z=MDZ9#?OD){Q3RyN$>U1R^b^0s(2ZXp8v3U^x^$~|D*Tu!<(%_{uQ%byS<8HIL}l+ z_HY<-neY1#CtLHXEZ+e{>PeOGd>VS1@?8>=l*yT(S%C6mzI|fAZN;HqoWjwyW>toz zUae@gOLbDNekO<)nTPY*I@b?U(VuhES8sB=^)e|uaYhsG9MW95dW+mNZo%Mk@uWrZ z{0dALSFU2op0Krv)T8Guk_IgyBEa}+_B((236QB4-#a6d1+RH$r25qWFHxu$U{5A1 zr@*OLBn}zm#^hPW`S}N<@m&O!&&3aV@E!>Lx`& zxnf;ZB}gIKe}3z2rbcc}=R7YWSOtl?8Jax{X&aKysoO8$u6f5|Y1Wr+19N>jsMBt@ z%l)(_n;x14miJq^P{j1e8@VfA^g_y=DvsqVYD_C;;}w>h-S2VJx@Uf8?`POIb7<9R z&mHS^ccjF(FJ{7sfF%>?L$Ou_Q6x543?)kG7hwtyiIFRS0_V$DXC{zf#8C5?U>=NU zJapJzK%*uT@aJgW6Md*#O}pk)F0zxl=!Y*aUVS*}{rLLLtKPfAx34az6T+dtAHMwZ z>K}8HDR~Jw9!}-yq|TomzyJ8*#jEE(Yz}A~>ZuKNZXpk%Xa)!fuu*fmmgp-IQp$MJ zY#0$7A}>arEvFn?-&Kk!HF9w&Zd+Gs&r~1HD$6M>B_c)MAB90vgew}V%FdDNE?d|Q&O+T~hru%fwOWh6?x4%`_yZlNy{i0gW zPCmQ8@l=mbjjrnR!-c&*{bI#i3wwTC_Ts}k-GzAQ) zi~Z(5%8V-TP(c7KU%7lzU9)bQ*Q=^(Ws$Xf?Oq(Nv-d31SY;(kvtqiED_IexLAhF< z6(+`Xk6tnBD*Iz;D6ICiDgfFVb`(uj*0!(M*{O82t%|EXrIP2t>vDq9#@3<|Rz>cG zUB%@qZngy7;Z2~ZyV?F*qC?EKrl$WvMRQ@99el38|EuKV?C9Tx@xQx!dwb>h z-`$-j_5SZ|e6F_t>*_GYLI&3Syi=#Q5&k=Mh<(u>V)2~(!w>$gIK_WgL9c5ciyIGl zovQIVB3>)H;T{CMR@7?U(Si$x^F9Sn-hFQ@9Xjd$c$L^m8?>iwlf^NVYY8jNT?K0i zE1Z$tb;1g#0P%N%jn{<#=H3Xn2>(5KGIjs+UXA~5=W{3c&xHR?Mf9ytWUnDoZ-LB4 zGXnJ#&65@BD@V=EE_Mw;bF(uq#>}C{0vFeuz254b-Zh87=?Snf0Ink4ZHK?9Dl80s zn@+Z%qANtMRie;`8n{;6jla&Wv+9-wQEP6Z)nnEY%k*`-6kOoOz|F+h>Q^Ib55C2s_R2SCP!<3(is&u&9epUZ_wZ#T9g93Y_(- zsFa*bH+#0r_xkuxiy@ss6%bs6|2j`f=RbDayY2lN|J}ytdgniiG<4I)K&EKwD&b&0 z_xyly!XU!8RhBb`P;v=zvw)8A2E@(`#>koSC%?NBefhFDKZoS80*C&S^#M|L^P5sY z#Dyn05t$ZtZUS!_zPep=JPll}1G3TuIius&er&|Jj()Irly;w^W)qe8%H-sEEl`uj z=QyGcAE@rzEwt(zz9~>sy;RI}n`F*0h!UTY)iIlf_w)rEVYYW`&n)BlpW$+w>3hd? z3C|@gZ=%hV{ZGN-*{#`fde5cpvu$}U^C;km89lOq{AN8c;$9rnnh7M#1-#4Ph$TYx zBPvo-xG8dp`23||hFgJYoMCQBm_u@E`)A9UE=z-t`g7->E8{;9kvJuuzUSQwcsRog zxB&lk+6U$HzfX4eYW}BN`D|zSfFNc-@i4r#%BW^;WQ;&-E5~uRyh^= z8I9mp)&vJiu1UYXVk}mlz0%3-FWPai=^?`@8h$^V?Q0szZs{) z#^ZlCHqhTOrz{ib^~+bH;U}E^lLTqQry;>Dm7KGGHvDrDu#mLwd~q-FS#kY;cy;*l z?JIvAu2RPQ^}ln_+1)9v|IU-WTL0h5XJhJe??&U$+zD#*8y4y|fAjNi<}&2pH{pmv zx78Zbbd>eMd^2|ln?zgc?rT9(fwH7{FE!jpZLd~Vp(HpC-Ak%vnyJf|S&VSuig2ag z{YK-pf}#TbR&Lnb%IZm5qd*%|*tjP60@`l<-qg_^od`=iFqr4KGohN?V zZ~L98TAXElqV}V0T^5vT`d?jkk_mHoS3p%rC{-m1Qc$qy)SPKLU7~BNFXD%`X-5Sr zm5lQ4y3vvd@OGaXvgp$V&6O*LWvO%EN}Ze)Xp{QH z*VnqE(`oXT)2&A1#F@BqA!@n9iC)uTF7mJydIPy^_0;*zS!ntS_;k{mD;4Lyv4u32 z@EeVd4fHX;5udsJf8OzT_W!fdI8HH7EvY9kCD@lLj5>;{|!8ip(-03q&hm zaX^w(v`oBLD`BB$zIfX3+EPYSpO6@FGG^x_R8vHDiBpfmqt||;aVQX$W>glX+vxFQ zZ6sy$n)diHTtW)nKHx-*tb2==3FJ zq8C6E>A6!ey%a;F5{WdGlYWGwLB9G8{!#`a1<+&n4^5Snjb7yHfO7%+hWuuBD;i(XktqOuMK6g6IE4e}=qq~l8IO~Qps$Uu$dezW z|MiOBAi=3c9H;0j0<~@LdNr7}Pk`V+p<5E7uNpo9l9PEjo7^TBbxn4h!$5`ERqUxK zAYLhxF0k5+*C0eenNxiw^{7y1F`6>lE^li(jtQHP|Dqe#H8%F_z+@^^ye{94wMG zZsCOb<8Y%GP*`QZ%r#dW9BzhJ!&TQ_!LXKdp(YhmZo8mEX&!~P-MR`Y8+~U}%8(qO zgoPqs(-wT_2o9hjg0~d@S97a4B~{#nnz;ZPW{igp0h1$gsMa+bK!w4c05-dWlb`Y;Lu%@oCB7Xa#qGNPdT>f5cwv8+@tHVZ@n8BBV z8!#SAG6?o@n)*nfSGHpqoZ|GA_{en~;aq~F3OPyQoV#WVj+BCr0?@Lwx;NEqv zl)>etFU|vV-K!4-s?+Babf|dqG?zuEd98InDjCykmRjd>w#pVB6Dc6kAi_gAv}%f| z(aM>^<$UEtc@?zi723(5*5#?zivTxIt*=8CV2r3jEea%sGS#bCL6|8LBPpw0KZlmv z=Tyo(Lm?%M*abnUTLHR$Zu#Y#%4J1X+I21 zkCGckYtG#J;_yTbsvSUDoH`|DLapm^fxuOp_lxoR7@rXdP4U_fB#b01dKBh&*DCkL zF4KLPm~M!3>!TtZA|b*>a7{wB>vfgdRa`QAs_iN5XA^50H!9>59^V^sel@}1-UH4qO1?eOqdG)KS4agAaOWC00QK!o3qTjG`5}XoR7s+X> zCe{iwaqd;QOV#&`Opu|$g}I!2&?l>@G})LdwR+X%xA-dG;zV;jWlk4Bgb`KFllAq2 z4Np>dVeY)E*xOiQg_t;|bT(=t;kK_^Qnk*GeJXpdPygur~F+4F$JQrh7$dZj7vOtxSh_9rfQ z^{s;n1Q+BA$)ue(1BGfUuL?C6IYzB#Nj6mjm?E301n$OxPvcp=dn1+9WTUg{Xh}`) zu!=d`=**tP_o6S01e&t4B}z?ozea)ulw=H2d#>HM8ewL`it*90GD|};ps~d4dEzQm z!wC-6UO6lRv$vsHA}!E%%=T+^giJq65XhCSOYpUOR{YhUt{X}OI05Vvo}Pm2(cXIjKr50uUZR~KYL8}dV4Rom>KN1m$9 zQxyqkqGYLc+mwj><=Cun`3UAj9cK87PJjEo)OpkQhPhrOr}`kf9Y56OQ0)OYQ%a`J z;*jtNyfm)k=!k_S$Gz$c0G(q9G6S*g7O^#Jz$>mTv%d6f+$XU03I17GBd5+IEVdD~ zGY?`Em4b0!oAP=Ub68?gP4}uRVA`{rW3Cr7&9T>w0VARI-bG3ZS_??e)R={^4XRY= z>^3|Ss&9A}U%M$&o=CfMsQpQHD-lI>Om)~r0n#~BEEj)OS-V^=DtGy}ZmxU?OIua9 z;QUiw%BlPu#udm8#LdK|fr0v&T-?P|AUP_p7Zp^ZoIj2SP4Bb1kzYmMY3|P}>{glh zq1+r>GIzC4!7lPpAy80e-C5ydd{R%_Puqp6Tx~2>1*?a0nv`eOK;{`Kky~;(mvUTU zC^zXKhml)(=Qw@J$_1r%`+Rj%zbZT& zK=_h6f_*`wFu**75yU}CxOnWNM&p=}JmNJE=3Yb0Yo6Jb8Va?650L{90+G)r2N(Gd zA^5%i%!&UWay-B>ZoPnDoq7C@i#`j^|Lp8^O8noQCkF?0{Qs?dfNDgu-8B%xJf#C1 zfT*CX714mif^^Ygf`bw9cKmi#qWDa7_5E4)$_)?*$L>`M8IQVETFy+g5sAj0ntX+< zJ|-gN1dq{?K5w&JcKY82LIF;3#D*pmjsX$E!gDS&Vs7np;4-*Xr*LNZ7$oyo2EBz}tJax9|0zJqx|| zvjZ}~yJWA^IcS`b$pzyf92dwtN`8SlG#oSp<7tNTQDZETiXk2J5E)|{bDEY5|ihvc;@>D&_(~dFY{(w^wj&b~7oP;Bs7G9AF2Hbgp zawZ~t?vHTz$B2^iiYongN{189+)`-tUn%1vCGiE0Qo{W(r$8?}XsF-J1Tpl14R4v{ zH8+vFq|-@4x(X&Ke|*EPt|$Jk-|5td;SN5F^}mC5#}6&!TH7T!kQN@rBNx+=Gjx>iF6W? zt!>1L?^wedg1G~A02ri{dO;QWLfJ7X87ERehU(Y6t2Sm3;cgV@YR#k0%qKhWgz=TK zugbWX<Y%|oB~U!gTnP4iAY|Rr(+_LkAfT~ z$sr#zp8uX#F#nBwPC1r&!QKFLi>yiU&X(VGue$fOnBVmEEbtvNb83G5Jd1?R6=MbS zNB?_51SHwmDDPV+PB!1AxVtPQw2=`G`1jpOA7MM`QOQ8q0GV(md_2Z~uozz`$|gCg z9_Dcu$^YRC(WQ9Y?RIu{_x2B-Jbl(~x4SY;i{dd%0RX4oh;Y(Sb;DW8;BL86_(|ay zQ1x0mtl?cpk~rSf<8#C4Xj|{=woDM@K9?%u*%-91*Ugo}t2S(B>o|jSVP5 zrlavtv5}S*)tJT~oIZhjHu4|#8-`Hu#nHzuY96$k_LsM0%(OB;_y$3#hGaFEcP6T*|LFo zUJ~|Ojf90SZBc)NqM&XYaH){B4w_B*%b3Q8=a@#aZ)7iQKsS7hvHRo9Ryq46vN66$ zIZnxNvRbR`k){jA&mx9H_(_v8h*n-mYr_=K$bUT6Kglsx@cRP;4zQ2lB$X3i8g@j8 zBhLCDT4Y^*BAV!5U31vN32luCj?$6CYPs8Pw;PR!o|9|5uWlKqIC@DUJUJ!-i$l>x z2l9zDC0Ul*A9v-C15BfglamoAV#K0Q+N|oxtkPj%&%|*2~8!#Zyb_12hj z&NvB;#qQ;vjEz$C`A_PJ()cLFLv%4>B6pzcgLwKZm$md^wGOqt|Ez(K7-79+C6Z7y zNL}=L@Q$TNoCqoAl$18-j-rW0{hcKRlwsg<95QeSLWRt2 zz0I9wDH#c4(QY&<`mwSXoj%-qvSWVG*UG712JY8}oma=xCBPQqk-59;mDH?QdC$!7 z2M)2B8rzVw#87D;zImgjfHawoI2%x7h-s;v#Znq-i%X7!g28J2n4#2`cmK={Hig5@Fp*>j~cP^ zWxdAHAVk#CNkrCH3AC?NLP6mNJVhI5(}Zbm>BK7kx3YcJ1(xm=Zm7zeh`||khkZ*a zENMLSVg?>tE*3PweW{hdB)j|&j`Kpv8keiX!(f3L$PSO+m4g#uBC`D|>Uq@1MNXoTz{mdN2H7Nb7HJQS~&(mp}PBam7G{xmw{iT`vT zb^(^r8PjDe;drj)0*TLmhg70z4B(PJ*YD*30ZL~ z-q5Ree}~K3b6IL;HzBFDVJ0b9Liy7(GU+NQ*E=HgyVp4^Y#{?HO39i3r$r96_gH}+7f?#l)ntYDz5Pb(e?UB zJ2Ka|fL1~@Eh58ez>x{UyFXms>)jt=^;Z0@ zcGEtn8xfVfQds(FyHTWg-!=DcZ4`dq>pbnC6J}!-BZf57wXVFoXE8Yl=X)Bgt-H!!!BXbmxvUoNGY>~_c~06ve-wjK6g=54rNo$ zpk^8*O}IPO7NtCv#begmqw;#7qv!GT>GPLx@>~N~IJHo9Hb&*M(#d!OKBuswHly;J z1vUy79^|fj1OY`-PT8B~(;~4zfqcuxIHhc>JYqCX2|ve?oJ+Du0jD9=+K`hiDgrze zO?IM8HuDZAkR^?WX1b~-e3<$@buTIbZ>3VvAXkpyiUmlRtLHN^2_lBi8Xz~#GLQ>l zqjcQ?{My*a1Er*7R5tmDiS#g{Sma^UMt+6mILU^S*Z~VP>7r)G-|=_-Jp)N)%Gg|$ zvY~ZNb3B3aw10uPc*elq4?mPu93LUd*tS?z-otKw;1}vR{T1`5E zMFmjCF(ET-tA50Wt)2ExyVc%nwRc){EO8AmvFKQx+osf_%l4yz%wX`n@VXT(-#||bM^xzOii}a3DZb;H~>@K zL<1CMms)*lZ$o}Lky?c6V<}PqoIN!|^jxPu0ik~c45tl!JFmSs2iBjb!>H7GZY3=g zDpvtw5LOOecljqdCw!tJGz->O&bN~ZGRPvu+$yD`lm?nA%Jh{celUv5H$H5MW8EpO z8~j*L-aZb_h`Qu%!@YM<%X1NV<`{T6g<5}^AYNVx?MC2v$H{(sR$cT|SI)1IgR!)f zsH~k97tVK+yj0bO5_Gu-z|+UORMQLN=%tZ0QtfDO7@%1-9<&E~M}V z=0m;cJ>===YW`dyO!KooIz_|hrC*f%vcIagJjRF5|MLFBPhi3R)jWhiA}?fPOylQ| zZAOfpQBuxGj;@EpP^#SL+J0&S;YYhGtqzke)3JPB#v_x?QiF8CY%zGwNVa09c^9PL zFml@tZ*JbeMEJY9=sjoRH3kojr{--=o1_V2&5oQsL;`{WAYgG!bty3YHu7q!1E`)f zRCO<@`l`Q8qY9ct{Vkyt6z=9m#z{KyLdrdruX>$m-&JmbtErz5F5tqjfM`yVBlAnD zdj-9SFdd84j9r^=s;i+Eqf=bI#gv=})u2|SjN>73uwH1LD$2lr%fkGA!4Cw7zjB(A z_c7Rug8o6Iiu*}6yVSaPtj_a0;T1=Je=+NQVgn*D9d}(+n7a%WFCa)EnUh6v=Yi|) zF=nhgBd9)Fu7*aR1j&tRYA?b7!$eENDzZnvL`0+!vas(Yy;sFK7(!s^cMjl3oy^thZzO2YDk= zRL*#0UW8NX#`hz#u^bg$ras0bhbf9&*#Kk~QpBq~wD}-RA>e0`@CpvEkJv^0Qr};w z`d%{!ggC{BUC10qG%3g$PT-7fibRTenk5b0KhC)1HmRrfPqa&oXR!hJ2=hKoIp%O` zZ`zNbyxx#?p{A~SlxoP1q45td@DN9|FAMehnV9t1XMNG&5Denoj^Al2=c@G+FWg-$ z*3KR?vuDfX@TtL4=|3zjzydZ-GS~prn7d^fTXr|!&Q{t|8-DqsadxZPt9z`gUEw%% zE-v@bi=W^B`VKZK^tQL6cyY;7z_^q%H#5ZUR;Fp%i%mh~f{fV3Rs*)rRh{9_#)kU$ z2n%t+c-TcEh#wn9li0~le)|Rfdi3@1*EcV5y zDbcz)g_v*tJrd|83(g3a)2p$O-*hiAY*UPy>W`Mr1iHkD4KPdwd&J;f*Y+*EiC4EF;U8a+!CW+9N0Ae%@d~zoI$7nNV=6tw$DwmR6 z4;<#hj*m!4)h0>vgl8hv*Y~hk>MGH+^uR14+frm83^(#18rv-#byec4e`Qie1QE^z z$qDgfDL}?3tA)s!$g6?@j*!qWg>OPSEHsg-76Gy}s4I>H+cs>`NLnh#M~5$783O;Z zH{C1rc~uBr?iQC>z%HB`IMlR#mP~1FcW0~ zd%oQ*$SI^@pLWyqxEsp}JvJT6srnjduY^!Hx9K?6Drz^Udar(Z z{r+7S*;%0*vG1cSmO@VPQWLq1rNlBcRshhI(lOBnpd{tH2jzDihpm%^Gl5NfFv5Ly z4($d)MUio$=wZ}&t=xmo=)o{y9yc0huU;0@JU9E^%}V(w4NAhLsWFR zF`tq*m zSec0>f6qJ6k&K3i3%W=jz&CsPF2%Gn9U8`h$sI3)EVBG=iVsw%8xzzXPiX>72Zg90 z+gRP=vxH`L`#UBVrnM?0dX@S$^j>gI?54ZGQpGpFd}#t9z~qrM>-23kebBfG;n4&U z5dhJKV3vh^Vs8b4ZPtVn%@y}EiN;e4U__#^FGgzWkJv@Z^bxIIN)yq;ao9^a4bDVs z1Jo{TxTQ(t4YF9dI!Ju3CcNoW<7+mX!*Dc4L(Q9-6R)KDB5}$mW)IV3fz`x3Ba>~# zX@q0MrUOkc9O@`N8YZCB09a!Xo3~}2Fo+d$rZCO47cckFYL}cw1#qW68PLyN{jd7X z6yrqY7rHcHF&(G?y9d#$R-9a`W7j6YcXnGFwn`*EFNtwIfG>=UoEpQDbX_mim1JbfD+oVsLjEbTYv^n&noLeOKTNY578P#zu~q4Gu@ z!?!ch*{gAbvLL!jEgpa(4P(m+m%mIsQ^Rg9n7K+AcLnk(#aZfwrf%P?j;=JepN4i4IroNCEd8+~tgL^~YoF`JAhacw5QWg+RJy`6TxVQ!9i=X>B@30|t>Mukmsgm4y8 z@b)8-mTFo`Nj@i3?TFd~98VAS+U-&@wc&5O#}3GgQNTTJ7+Sr;QUm%6vwes2(Yj7N=E%mrPdwEsn z)|s1IXK`*t@l_@M^6YlXefLvU-wC%zmh6n;rjyyf4CJ@PZ8to?`J~n{JelAvF~XdL zE&E9Mrw}FeVKp-y}Lj4yX9f(w%n*@=T9Y}OnT-1c4J@=LR9Fj3B7U)b;%u< z#k%BRZEoHf4Xe_UP+dK2_Sy8$ax~R8?naJ!OcT!&2k{@*7fF*$rz&0^xswh0nemXOH@AsQ3+>C}+lpufse+ zjwT*``GR~!B?l(sB(Yxkjo?IjRA{7DelLU-hgtE_S6qfiaa{~pe>x=q1r{z z>Z$P`PkJ=$bxQCvrKy|%wjlX_DTxHHR2?lkEIkU%Zhrw3d2&o-CzO*H7u-i#t4Cg^ z-F{S&PJ*RqI`2Y-MD=r7109Q}&ofjcL7=EP9?#6gWVJizxmt$9bY>E>p)s!z%?lNW zGt&Uu*!5ddH6H52Ntgv>iCRWj%xylaKwZf>aK@9{HT`UT;RCZ0^JX%Zn4C z7wYrpWs6IF%mR8V+%v?b&lrD}vg?Sc7|^}lof!$|OnB3%F^1Fs#pR<91%A{E)c!DK zB*r8Hv9eR1jWLroJWhcn<GTBG*WOhwLOI!nN(VtW#ee_ze^;auA@cp zA@Gnik&iAfPtoONNwwz(gRif^_0eSIF!X{Ej$=6fxzVtt7+7e3a6w_7s)$Qe@UMAV z#9i>LfUe+|j`?XrvvpYr#rUvCLyJ0*BE51RsxseV8l(4o=;x@g7f!=Mce3ReO~oEd zh=t=G)TgI^`!93uB4A?|m-U>s&%*3lAlaaWYh!nF4qERVpC)jJj`661LolL!d!Rgj zq4gJ22k5Q8D(tVH8uQgL0rV4F@dcKEO;?%+kMsaMa)#@*JCp{jJ$h7}Bsy>iKKjm~ zm$KWPL)ue7+#O@f24ROd$$h3Xx+t^+3G+u&g|IRXJ1FnopSBl&*W{cUd8ed=*lSz(HdMpL$izA%MsK&|BcA&T%nC93gpG-3ljY)A z+BHQ%_QQw6Uwi-a`29QMa>+!>##Qe^jGTQtF4AHN#)$xar`xE zak#%ZFv+!!)ME%ruBu{z4cJ-mtvrIYl@gJ!( z{3Y{5T|5Au+x#_W>6iCeHO*yzQc)9u7@T8r2wAsv5CQ75bmZq^PgViK&ULSKtKJ3-zw{bQWpf@`HgUDUHFe42Ntr5s&PXX|)2+A~qD}85=s3;s{v} z>eQS1NyDmDt^Vh5#lV?n98FRh=vu+89K8YKJ$0dlutFkYLzQF<+j;?ug<{9l8TVpC zzgA>O*BuZOMK{IImBEXUI^oqtO^Ia(Xj9H)bGfuRMQyH}D*p~gUANHXrE?*5;Tig> zj0%;6M1r8p%SVso-*Jd8FPm~;jfOrfp10EuFH^P+!6x zD60c}Eu5tz%?8ohPz^JO@MxS9y#bTu$X8Fgy1?>b8U^KKSM&rO#yx4xxQ|4@5;<=& zL3kIczRd{^yNnOb%UL?=&EnCI*pS981~s;qHR~5an=SVycw~Eehu5Ib;;T5JzVo_@ zz*pr+3z`CJP~KV+_81@HSYP33AF3mc($FaWU=QAHpf@iMkJdHItQ`Bv1{#}dK>&V0 zfxlCx!vD>6I{7^|QHT>iW#b4@DfwXU12hzjw|r`Md`dhr8`FdBJW&F#r+`{P%-9l3 z!Wp_^N;01q=d!tH%6OaQ^5~67lt@z|O`-yFR&{mHD#;$Q%FVeQTg4HXwi>Z=9pSzq zbMs_wtK}l!FyTd}15E!{99}#vo*g+=mr-*UJ5(H8(Xm*Dp-tvr>T{$69xRh%j4P419v*}=*k8Jd>8W;3 zc%KQXuUxpC+?KjQx_xtb84l#O;n&SzKlHLt6K@Gsd^c)@R9_Sx3L3%<& zCR=UP$ko(KBY~bfV}g(nQBR!Fq<0SY-CX&m;#SQehq@cEy`Hp`w%=_0nn;Q4sM z_)z|#2w4h?KZTrkduP zj^3pq39vbC9k2+ldMm^nnrJg~a8o&0)kd(Ku|jj*fU_;sm!GC1GR`kqp4_`k?M2}=6Fm}!bOGQzwPd$LROBoFI3CB zE1awR|Ji%@uE=d{QGEWcPoXiKX~?BsygG-uJ!c5e9ezX8T%eOXt4|IXTUF)YvK&iQ zGy(2s|1CW&+48HZnx>QTT1iv39$Q<|*4|tDaU0wP!zA$AERa2~pE_5vye0jNYh_h< z(Oj$%GuTKrd`Rl8$Q)c(KrY@0ud$P>u+uB{Zv2Wt`&IxY6PF?8iL!ZQIG!S#oz}NO z3@CYFbVPCnEJV$O<#r306AjL{q)uVJ#Xm;DZxi@72q6yurAC&Cv@Bwz8PSuaLgxex zK?tWXRBmp}Ha!f1SPN+kHWWdC27BP}=!cK*Ho)=e+YcMy=l%234RCaR{^5KB9Go9t z9v|$VbfoVeGb*M9SzjRL*-x&@Nm5??uMvX+3pe$&8@w53DDq}K$- z31S{`Zb-3o55#ZigGuC#+-L~>bOBQ~OytQ02^3bd9noYkj8Fpic{^va87*|~il8de z5{9)Xp{4IXi&d#o}xW%_a_n7))6hLH89Im%aLa1 zmcBe5q!gAXG87ZU*#ivS#2bkPM}g%-XbT&5;qw<@GgLEe_yZ-DcR?=XoRNuE_7$+S zvk?zcCGs|6?xV|qB(E@6yT%A}uMfIOg*njv$qA3!$UvnGQy8WlVnH0}$I~=OUNi|q z(fWHxMm$|EGbTmAhlHL+sp^C;I@|$2(^>m@|MK9+JgR&IT=7lP#tlBr7bizYXCM$9 z7s~=lK}=OA4Dk`bA%ro*^=^-nGZgulqktKza5(pPA2RlU8njwl8_K;UVQL_8zDO0S zdbPq^y^*lcp-G7@E&-fjPfbA9VL>|3d>cSP!B=4+r!ZN zI~Y&A5f}wSdY%etDhzHR*xK!>k%P7VXq|120d)DKQ`AH#Y6~Oa?=MOQLYv~B^+E?s z1XP6>k0e9ku(iy;Y$zs(j+(i^iSQSQ=`z#JbB`uq&uO_Uu)KD(jj+bhlL%iRi@86T(WVvW%^AJ0!joXx#%mm!TZ{4A6Yv$JFO zVdgcMv^MJU79`EvzriS+kTml@9zjZySS~uQoE1ivY{?1JC@^Ga2yI9N``{I{El&^Z zb5_DU%4GbfNcp0tC7D3QLVywk_{gJsd8SEit>)%W7m}Qa!!-9a2T{@u7c+NmJ$ek8p7yIr}DzgEpKR+=FiF(u9M^^zzl%2rgbp9YQR#`U@Ad5fWK%grEm? zi3qGsB9>Fxb63Vs0C;jE3QQtj zSPWnDH`hFRz=Y`V@j)dl#sPsYpQ0SB21}tB4a`lf8?ad#gVjx4FlBHBtg%w~#j#qQ zC9hS)YgH_O5TRSnWvAZ$Lh~$`YsPr-B`j92T&lywjYlk=m7K(XYvCVQ14>;>?jTCaOB0P2!dp?P3Z-*uU~ZEN_sWMmn>nAmKJxwl+1RGTAB%e3TiMq4Q$~ zXR@?zamIDw9$_h;bGZ*JY5xv@|f%m%TPg~{CF%Q zt;$ee3!FHWY&H(F1w(7ccq zAy}y9iUbQw_>~u{ku>fW>PQyK5v{3cVfOP4s3=pAmwL)S5bwO?o0fdjyn0IAOLk6? zLNzq6OrephWU6!n;!>|wOMPj0xoQBHatjI{P1pr#dAR^qLHf(TECpmG zOCZ6n!4B}E4Os!b{cnH`5b*$(EWnZlSh4_17C^E9x*h(^$G;2{GUw4RNYw1QMC;L9 zb}}<+BhEpdEy6Po?OAvUc`%gn5fqFw=^deup@bC>_-VohXx^LiGXkoyZ;TUAaM$4l zls9iPZous08{h{dz)Ox`$q_6$f+a_wI08M5`y4!h9&%xB&Oi~Xh42Qdwddvx(nF$> zH#kiLQ`F=Ra-*u|4_HDHk@1&vO=;+PMQ&M`4EaNrR*tNvJZ%9Q02KP*oZrfDDCvr# z7^4kHANC&uOi*$!KTtC2`|~`7NG1AqabJ7(~ND)@mF?=SC$KEhv4p!(Q<; z%m-&5_kgq8bF_CNNtk3C#5G(ZlTW9P5-Zs;9Sht^{|T{`OWY}BKHx>sMkh93GHo3N{PZ-O(U%Y5U#QL#1cRT1}R7_p}uF6 zOITL6HbrJs=@!Vm;HGrK*RZS=!=K3FVI-+xaU0tFbpXr=GvL;j;9`da~^p`TMy{nPph|5n%#Eo_mqGJe&2txN@8X zh+>b&>7w0>7h2L!X49c%>@GjT(o-w%5^aG+g`GwTKj%47MTuBwuP8ZqRiiHR0lc|q z5=fdkD)P0_s;2kA_r34?+Ua&?zGQ3w=WX1Wt|Edw6pGyZLUysk(lK*E3TH*emj{>x zEWsFWOeN#)fTPcXp>?QKL(c$rF4@F=URR?>C8^ct<>nWLow#|Z< z=@vCGJ*a%-b-cvK9Uo40#91)LPAe_aZZ}0ihvS*mG$*k;?f-My>-77b-U>aa+I-FO zH3);{&^uX!M+;{hKJMX!vk5Jb|-Nu-{o(uK@%!t$VEzCZCRCb**{ zfgTCd_%T9!cf;_;^=?&0=QPBh;PBE_E+%LN97n)&v9JceVzQY?Mwkc1NVk$<)n(^t zXBTEl3B&v@N1PJ^kTRL))cH)&CIxJy1dmJuf={Q&V!+-)7`M_WS~&>>NHJeEQyj1n7Lxu&T#A^ov2Et9nBdGHZ87qBle3 zt8%yt#jncT!mI$9<~D=YH8|{Op~e&yaw=K_&8-;R;rrugZuxB>tM zwi}V4E#FAM@P(k0t^O0$iz;eXT9sRvT3hXD3AhZ7*};9!>k>m=oOKvOr$oG-;4-19ig!l zxu-8j5C~{lux^!_y#sz``f(m%K;!^t1dS4}G`^5H-*{_Ae@xz8wGLp>%Dk9?;Ofm;!~(LGS6qAM$4Eejj)Atph@Cy92ruCN?rD18xE$N5AZt7U>ZZ+wQ0aPoUrGyqy6xpxhSyieUhWnE=K)8r@ideRZgvDs` zjj~57XuDRS4IY@Th3EKHvGl=lv>`5{_P{6E+%Hdx02ae1#Q&AD|3SX7r3h!=T4q#0 zwzvoLw(AF_#~&j0Pm zzuQVdl6|wiz13id1PR6-nv8(DSr0t@uo7}C%8pDgQZc|F{((--9{Aq}$A{FL%ttS>!k3*{DIk z)87IS)L(nG~_PMfyp4lKvpNJkWcT3weSeOVBWwgcp#c!M_irgSLMkx#PeC zV>b#0RN4X4z?FmN3C%vR)V+GXDm{j|qiOB=Q9~KvXCJuSwtcMBt*rv@5&vB;14o;m9!MnG#bWdoaVm0yoC=WQ7-%}6vLl1S15 zXYzY%g$gn-A*bjBy0_4mSvi?`0FTfJWaeLW8JmgTD8jIfpm&Se?Jh^jinMQ#>51Xz z7~wQlpt^L>1K+8Of(|$)y1wV;Uf$|=S}QFo9*yCV2Gb|bb1TJ>YFVn`;jybaU@A^B!SHs4mwGx+y~OcqRGc)vl>#?q_>nBDVp zH$F2J2QN?$pi%=NA z?X~C`N2#wzLNJ6xZh&?>WwiAaDJlxJ{K}K0fNaklGv*|CRIxtRa%~MJVMyC^Kxdt{ z)kZAYLF|&z8|R@z$!{muz80)fYuqu3u>+o-o*eK9cpd=IZu`(fJ_LUU&>NvvtF_r- zihxWmP7zX=%Rta)#~I_4G%(~14}1UgB=y2+)M3hMGG#KCI0G^YB57*HcwLTs8+J8} zfTyQUjS1-Bsn?M_G_!(g@o3;hfxw^GHHWY%y9pv+=ia~XZGbq0E{5D&8fvlLwdnOs zR!yJ?ddk?P>+9Ec7d_)TRrCv7R52RJ|ia*gN+g121v$sKZA6mdV)@BE!hYC9|Q~NrI2(Cu^7) zEeiZTJhVaLMnhN-%ALSik@Y9zBB-0!}|19$lS%IKOB&j&9;$cgdLGc@vkaKDWjrsi0fA!%CnheLP>;K?}ZsftFbZ&T1lCp^?JDR(%X`yhULwQlV2)fC{pqNA(Lr_sI&4mjC4K65~= z{dxA`kXb5a(8Kg7jpru>jvL1v^HzkGdQ()L*WAe0V#GP}Sk2BkWJjCKW9k%cFezKI zw$+xB18X`S7PMr55+>TON=$(*p1mI}llq94&B0L*>l`1IW#I7_=`rW{;k^8fnRzcNA~ zf^A{QIVYw)i~97N2r5R^Q6i;En!HlGXU|C`8vSDY>BkT`7A zN4fmh-|Fr5GVOn!SD}Ys37P0#X@T#$f}n6sQc9k9$wQa8 z4m$&EB+ROU;N*<8xypn1@4W#XX5FaXCkPStS~T6}T;fDD1lqRx&cmThmAOg;Y z)`8eHdh#dm*h=}_RT#qY2LI6z#kqkw?X6|W8jS4*io&l z8|7pXk?bWcCQI=bl)}GhndhjiiV5A9r8Vg<5{GG#)f9-77tdA6wx}>xbe?FayM^}~ zV3k`VH^7^TGG3j1|3u-rh46ex+j94Z~ulL8nbKxOgjCHvq^l%-bvRtDh7yk?1 z%Q_BM&^YV7n8Y~G3`wie(XA2mZz`XQI*%Ztvl|hxxi0o*epS?mHFXu56Vf6e=Yme< z+fRYZ0!FD;4y=|2(wd-b9V+|sS{?cVtXshp%Ww@P#W=8kc*`JJXAO0~1Njs`OaUNt za>~hpia`#i-<&2hZH%Kxy9m=TL9Hp`4}{U8DDm@(}lac+khxf)F< zGE+u%z^Vn38q=d7Ok+;2YKXSAfYa{nxLM5>RT#JJ6`o$!vR5Wa=&XxV*_$LGXye-f z`0o|~?Y1y9XbWFju-g+K#_nhKQob<`elZPc3Re@+W+^@KC@(E)UwwITqskii{0HFQ!)R#kPKRtEjnWDjIAEXw{#YA+DDJ?4h2r%@)t8Tju zI^ThIyZcvwyG~lp+3S3_VZPAuX*B3;bce5Ot1S;!7cfNc;P9-n z5m&Q4vHJ$e_oraiKH740BS*Z`dp!A+d2BONx2-B zjoA?csw0(U{%mu;#}Mw*=)AP?8D#wo0ZD~_6;_^750NWUH_+*~(n!_9v`{*}>X**K zyKiDa5@?Yc^-- z(v2Y24E;4*F@t8S=Ca|dYqn-qOPk|2!pFj=?t;fpsokozwswQR)k}L4htHm(H6>B9 z8_iqkQjngCt5H+X!ytgEf6Oz-Zce_x9@I z=-}h|@#Vj-4v#PPe>ge1IzBzVJl;RK+CO}Me0p_oezbpi{NZ%ntZXyjSTAp%w|BOG zaq;tq^TVsFx5p<(jjAy2o*Qqlbl3Lli$`#?yA2}`Jc8dQh(JxoHwsH*yR=bxX_9YP zl`U#qGANdYathPb@Fc6k%>0?5UObtyX)^Ql)KV#g_oUv+s2~o2)nd@|>*{OktC!DW zpGNmTngwZ-KFaTZHn)1a+u8e{&DXu<{m*keE8v{30OP>maVDKY;9?TTC?SIL zm2ZXdlW>M(ZUCHFU|yv>FMkkCK6Cq;jTSA!E#moW&A(K}V4$)cP949bH>7YIM;PiY zwish=Aeen+exjN~E9UqW_Ti2|NPxNsQ%PVdA}zAI^aUIYphxzAbBZnm0Q7b0L*N|! zHgQ9zm@&}6z`|?6m}3mbQ56?RL)rodYezN#|da8z@-)&bxq!KUq6B zkhmFJ3WWDKa$35;)G94xn{}q~o(AJRrjQ*3A*5XJn=#$c-U#kVai;rS>*#YV4IFg$ zd`9RxtwOPfn}oE}uWZqD4p=qDhN&wJcdX)vfk!}+N#uduE%s+HzL*RK!DryKoxEbY z`&}#bgOKU$CSiCF=+yB<=x!V{xI3+%A#a|k=u_3FckR$ znftk1r2z(O!Xrr7nzBUkG4$3ut>XbmxSb1cMA`K4s2S{CK#TyGEY=(YUO-!j*wjqP zLQmvJg-=nW@OvG6s~e3rSY@#9y{SK;DU1pWypgOa5Qo!v*$}vtK(_w_JDIk4r}dXt z;T24}=(O3SJ4E^exmX3jQ5%K3hnCDLE9F>dJs*mWW&XD$6pt*hh3i4!{4WPMSI(?k zD7(Q`+#V5psu1*iOe*WrG>dgaI=l>hs4!43Xe-iIkiH<4WxDtyXG>{qz<%#mJ?cw{ z$s8kRQ(lDtr%h-trI?^_%#97Bk%Dc&oeH^IPD<@WkRWDZiZ~VUVZ;~(7A--u+)Kkj zi5PI>8|HAundtJfMOgP)(47lPnI(dS1$1l~)PsXtMHSF^j3N;~q!@VqokKo?;kbiG zTxzgOwSq7Rkv9KbvK}X**^WkWQ6j0uejH1Jq0H0A?htl}%P#BlR-03)etdWUlr1=i zL&#!Ox*psQ!;^r(#0`P-Pdsuv-#M0M9Y<3w<>9c4%XvU}`aZvXbIx-Cd#?;FGAlSuMhv?Xx zMdyAAOWy=d(vPcThl&yf$YM7HRTvYI8-i%4jeZ@hWwZ$Ycmw*K{(qM15yZdi8H8E8 z02xENS>QCWUD8`-TJm!WKNA>nYl6j2;*WHZPysXqXUGTpB1|_-N~U!byG2u4fg~*o zcz3!Wx)-e}A z!%B4*Esvqoo%XwD`F67lR~0T>sH#w-MH&oLkgfm&Ot|x1C0z;d1uL537yf2Bm~3Q- z%8cdvz8p-fGz7DXR4;dKnYkY9ZEWKIuMdfAf<_npw5i~K66DR_WODy zcD*ScW0#CB@8c%;I}=s@_;`DZlCeufjM1}6D8kL)4mghD2@8<2<_8`<{1IZ(h?6bq zfX;)8YLxx>v|hnx8x1KQQhC(q_SxJ#Yt2KE=Mw%Yb19Q;5#-+J+#{VAav#GF1BUsr z8-L>FYrn7%$^+)=_{8>Y8>AAgzz1hY*Vr{Rw=bjst*;9eV3v|e10s2|U1fT%&y=u? zI^Y3t?naP>q~ochv>aICCS9<10|+r2kDYK!{X z$lF61G})a8fNq_s_xgqfgPFp4`+2dZAfc5B6&&a6G@e9OW( z5dz0C8xBF-mqV;UrelmlH;{P?KqkDL8BD09uE0R4OP-7bHDIAABOH)m3c)SB*CKWl zuk`{izqVm+x2oFx7re)8+wZ*r*Dd4qUaKvPr#>-R^Tx%{*#M3bHGk0x;4z7dKZ*b% zv;arl)h)a)EW#w(D{D(sI^$O3g`Ke?q3<$qSSXR=Vnc+6AsZAARCXm6WWFGa{9dnM z=+-I-BbWr<6{5dtlt;J3tx?SVeUd=euLFR{L+GJ#{n8O66@_X4vB5|I0= z;tG?9U8`P6nnCN1+^Yoo0k+Sa$T`5Km#%~>3ZT8Hj8=KY&dDpfdEFuG;H{FCSWxE< z-uUjcaMcymV(7TK2@zWo;^s?qc?}+!*FW2SZNny75mGCC$n0e?HOEm<<@wpRnS)<# z=sK-DW#|}hv^GTd-Xuw2MBckGtFJ+mD=N-m6p|6X1D zuf40Qwg1|K(e!Z;gz$0dPC~N2{$CHD{^#kJ|Mlc3uaF#!ww`!j12+i89p3;r7y3B? zUi}AtB_rxmfNr1)J*WlNO~XI2IOq8=ejtc2fv(?CQC!Rzcv|v9=H<&HQ{UY4q?alH2+8vB&b^l$TfT~pSGhxSxVAT%>#VA)%<|J&$?~T2@60z3 z1*SJ&+h%R~EbeJ!{~05{aqy2S`_KON=FU#m{`W_dGE4Jzz(LI zy`zQmb?7_(3mf8XMF%-R~E zyNE)qz6w2y)a21ZZI6~}bB9F9;Yj;)E^Ua-UTCA<(XAt_{c}@_L^m$i80xY*c;rgm z5KmIeU1?QQ+-0@2Q<bYBGKr&qe9naFljBv z0@wn@l(#|Fo45{6R0<{8P@0TvHazfD+wbWyh*MP~_50mI? z6a@3UdO(v`uuQd=SNUf}n(~t`6IDcxLOSiY(*ld6L9%(}M~#nb(V?Pxjc8GE=LLJb zs6ru4DhG2`y0l=p8`Gzv=JGU3(h2a%Vex|S$PtNt6PD}~IBljvR>6A8Vg$&w^S8eN z(HLld23<(Jw6ryBkpq6QQ-}OO7EdBn9`afNkMaP)XsVOFNJ#=BC_+j)q%V|;W1)S0 zumyCP1s0V|I}enENi$|MK$D5Pb-I)fowd$Q%vH{d$Y0haoj;lvAzPV;d6nTXl{K)}=B+I9n3wR?6Ee)wjgIteM+FC>S*-B* zwgEEhp*;s=jSpy}k+a;X+~`Qkt-LRG99dR?T+hP@C2&8a8|eNlK_Z1n4vyn@rytIb zuJ%vPkMhe$T`7 z;oa5A(NB4tJUkTAhUUE$&fMD%Cx=Jpx%n{8I~KkM2Pg`nVOL#yXBs*`y7=(%{GbG} zc336TXu@ipVNS|FM!|0r_%;Y3b5qC<>Er3~|M_^tg20^azdyRLF1D&2R<^`2bQ5o6 zc*U6zux`)xt;nD>UqzbE@qf)>xeemW5dQ>|U~r#@g*|;y zp1)M#BRQuwFICNRm(rlB70TGwEW?@^4zm%pjA{nXb53>qOpK`3-jW+#a-)T-@i|#i zV~zh|_)>jTwVBggrI*8@8lXK7n_AxG;&@f9pSjsp9p)FxvsP@u+1XY{D2>*dG@r7$ z>oc<3^YOM7DEpJMU)s#g#)BDqs~%r8vq8a39Fb~k$qz00p@LQToZOJMwqFo0L|LI( zIiYk!wfUe-m6yW>X@ETs4^-CS;y57H%iR1=3hxW$ek#`6BDf!er;+z(-wI!|RGuxi zr?1JqsbxJI@1{`&(Qjc0v5XZf121ch(2<*=(s)>-$f>QK@(mYXihRePo?6*=(nQ!E zi_36jH@xOKxHixEBIxX?*9s%5bHewp$qoF$^=|Ln#AiVQNWek@uuad42m|qxAi6cP z800@^di654we_yS*KAoUEOy3CIPT>=_>$9J-h&tO*u|IIEXn+PNJhxg*z%@41Jm-R z{F~vsmpA3la#OB0T4f3zp6}-4#nE|1Vw|d6JvrddkNyv9mrkWQgsc@Sx83{~yw8|# zY6hHZdX^bmCLqdeKGqGEtzhL3%Zrdrz|)yvnEZ>`;;Qh_X(k|NVW>}24J|sA(M&TY zd-xV-OvbaY&YBTdw0!1DSYb*m&S8jL%R#^v{JHHEDkR*rp^xV$_O|Ry!`TO=m}|)T zFs6bE4lubCX=!sKFFa&)j>P$S|MK8RN1v33Tp))7H==l);PNI-c*=;R4#9tDB3W^< z^%|!)8_Lzi(fLnD=U113}J&@B~oO9fp+1zmGh zn{kdjkGd_(N;Ouw74Ty7s&&fYsv?|K>K%L0FR6m7!gF(*c(c96s<>yGw;C$BXPmck z)m)8fGlx@kpSM)f{RxzGYKfHS>1v5I*40$CW4@f)uC7z4)flTZP-A0l-_k2|sqb5Q zg_^B3P?n{jE=pUrbRGpu=h1IR&9zwf(dK$C3jtYxmaC%0xx8$fE1v4KRt3^hzqHgZ zE%i$aFaR@l-$JawOyd7sx~UB3GJ|r;&il--rx= z)9lq!Qa$7BHMbzEME25*?9Z&B`T}7xitW4d5i%Moe3n5kmg=5m(2E&VJt?4zQS>Zh zV=QB1lx>Vfs&;BNy^NkQM<6u~6>}?iENv~-J4^M>QoZvG>K$$GEkwDalm6#X?PNHW z#)=&~r!%i&pcJZ#!=*|oU@iG26*?6tp8f2YO_}p7Ggd>9^K3KLT!~YG;-w1b&#S`u zQn9iu8$Avu!yuYjc_XSXRX0o3jdf8yr{YHT`oh&V63oRYZDeaUl}!nb=2q7b*W&uhVMIIKS zf=Oxi^C(~po};n)#m2GBE6SHZR7#$!#CS`&FR6AZMeM8$>})ERFFs2()Gg04OU+d+ zrHEZBTK>F>7O^m9Q?n=rK2@?9s*}#a$W2J+RLd;AAQeeH)yNDgkqU)Fs%X$);-lP; zP`e z1l9&YyK->r$` zs$zqb`59tOP~r{?Nc7ZAx?yl5@#`4OjWdGE&730u2`_n1*9j1saR#v`{9 zZ=D(ORjJac8d4@BuPn8ye<@mOu{BDqKbwtr)`7#d1AKXbhllQW-UxcPw6-@% z5=!6G8UT;bB=o@z1S2={A*HG=777ONT{nklPUhRZKtqmA12l_mbMNO)!#umv{NPzE zD08m<$V^nI7E1bl;mag#y51s@<)3wlC?LP~0+|z#IhTiwC!1Lv^7Yt_+4@-I^2l|q zuaYLE->S3CzOGai7*P@9IainL`fD$)`QTX8!Fwi{gE@}f8EJ3PnY*CzmquPL5*@xk z1Zb(00L41TqVUY%Im6zR!*y|Rmg7RLph;$P_T`{ACtJgvd)F$d$pY+q3D#@PLO#-t zeXNW0?P>&~)ldk033XZ{unOGIz?U+pDhAFXlkfXT`k^5ynBRb=)RiT-5o0_WpBw{thO7(+)l!@(wi9 zGuhMGhr_GmGgZfWH$4bo=;!XG*;_t=n7CxZW zZ&6khiqi3O-HNv3(~Haflas5nlaKF?PcO>RY4Q;uadjbR8UI}Am(R(m$zCrf`k2&lOOgE z{#9X>WOgIqIuqwH1*y(Zn~p6znkslMvbQybS4HpMUY+e<{&;b9xPQ5C8TQ%`D&nuD z-;M%jy%W1+#P#SU?Vfyim$QYf)ozG}b~?!0wNk;FO_xzw(fav^pN=n%Kb#()zMHKJ zO(d#wRi&M;Ke;a}A5JB}c{ruPF#~qB$a#T5#tf@8z)Ee5Ohb4WP3eFIja)Oo@GT+F zeOnE@o_(Lc+Y%r4?1a(;K*Itj_ZeDm-Yi?2s1ewqXo$6YucE>*nl_LT4TLsC0l!=~ z4>uH(07Y)dX|DXG9cTmB(Cnsoi4!;oKAUnLeUtVlN~~7iP}du@?IyIHGG5>O(5tLi zp3k=LIc@sFU;w>)FN8lLbc@qriMK{K8fEQB`;D1~w?Om3TH>d%UA6XWX{GHoi5JUlC8ZG(v zUaQu;RJKh}j6yWLzliA)J3tX8i5m#@SK-IK`lBHzgb7QZgbUHCm8pYSt#~8EauqKF z-f~T(h?!i74J1c)2kIrzC6H4}aZv5O0q{4A@XfCl4{{ZlFwWfyh4JP?J_^6y&)AKv z9kaT{7nA%?IZ*!VwC;mWA_u z!8wupq46kL^Q+v2Wxibg2D8FwXr?7f&%w5}!5PMuIYiuGlzPY%5P3yAqb+DRU zIJF0sx7p0uYnofdTE9Slk=a(Y>g2EUn{Cb_#xAq&+gzuhmrDns8z`~Fqd!jd>0h$~A`r~#)>eeB8(V|U zL!GgXTwFZZRZL{AlZS`ycOXJ!MCD{*Kw?g~4I5dlsn$Udy@z9z+-LPxP+Vk5oQ~yN8dL1`KSB5H`$Q-N#>^?JM8+w9+7ub2IIv)AAKOMmG&S4*l1?Q5@PyW6b+{+ES~o)>N$yi3p|#(UtC6AY98^|h(d@_FuO#`FJ%ZjxwNj}53e|NGlp zTbc8JduMZddHz4g^DO7T?NtAPub|lu_gRimv1DH`Yk(&=R4NyDwH#v?+%Rz8^QfTH zDpv_&qwuMDL82zDIFhkuoY}PFDWg+}>Hz|L1tVlo7}5CL9c+K@f`A9B%``@3Qwy-NIrq z&`$KTW7=;?ipfkg^2deK)Xc+aHKv;8;qtPM7mkgGN`qfB+JTtJY{(ovxIkJ^*e>gm zNDjy~F}t02P?`mI($>H_cmy}Q+c5IyI2xPwPsA9P!!&SJCv|^2={0L2J~FseZ};-!2`fa`0U?&or_ zix)Kmd^XUF7SGq=Jer^XV>hl zS=qt;;S{wN+lz*bgT#gv1HApiY@`@^&=i;s*Cr%0?f*u%Kh^txx0l`jn>(*Jm;3)Y zo{Suy<bmxbyGRFZDFp|C+t>uNXz`y7Zv({&y!U|L^SX^q2ep zIi80ygZ_tyLP@+be5;iF`WW%5||=23yT8t~tW=+Msv?NHWos$HMVA zFTJs7x_kOW*!q`vd2DOC93kM*0yP~0^R2UC=tW8tBLk@tF*PAIWXOj!drKBk979DF z+wHdg^JN@U%jXL{jn01$ttBo)rZnVMjDL0kd{ zlkftP_54k+#VcsNjtdkg;5sMZ#xd?HcHmI0vTs7Rut*oYw1vVKRxdV3UF7w0481+m zo6VR|cR~>M+%SlS+JvS)>~R=)u5{~?Kg~N`@zSD8Ng$7w@?(GrO71lu0aZ6&fLE3N z=X|(hmw2O->Pr?`>%=m3y9LNj3ETxqTmpw8o+1{%tOzFw^R+8eBuNq~Mxd=~`j|IH zGYm&_8?bq}A^*f!ko6o8a?Ffn=`l<55%g~HWGu8p_AQr{gLy_AJQ-*EEAcJPbY~Yv zjqYXWna-uaf9D4mLuQumR0y-qdO4g1UNoqzo?SNB>g)E(aK-j8CCvHFY>AGy_~3w< z;DhTN5UZoH(T>F(n4SxZ2}Gz*2lw+SFqUDanl%uene2y}ae8uav|%(&9eCyTvp3Uh z{oPJ(phvdlKrw<>vgxpC@T;xC!NFZeW~|FPT9@`FDW^nYu6vzMX&{jJwa z{_k0y`RIR%>M%=^ZDf-!sI-irot$w0h?3xUDz}Db$ln*u!{n=|gxxsKNY=$YXQGlx z`((DK3KT%}lU60)+t`iVA@tifA}IVh455o*M;^6>6@v&9Hxh$Z)uiMvj{=6-OO{AQ zg|xM)=gk|<@*`QoBu?RUF!Q20d^;=(p0+uWv&>%o%w(6xTFZ|;JCsGTP~*Wkg~#~EkIizhE_ zinPIerTMWzP%t?qQ*#4H?lh2Lh2^h3J?c(btum(gPt0L9;s4C^9qroY0OkB&zrWSX z+yA{@+W$Sv!<}H{&M8a`Q{jBK&>Q?rnwT#leVYSqcD-+_yO_&QnaLB z?2Vv531MPh+ZwOAhjXVm!A^BAx%d%USGjMC=+B!Z0l7aw5rLo0DB}8unFD9mfMJ9% z*$)F3Lv1>>mtV5)I0+CNX6RyUqC5AgBJOl$U+h;Wo`hj8mk;}NmVPmZEUpLjb)3y% zkj+G3#Vj8z-3-k)WT1>F0i%k$%d z3p>0#220hBD)!PT5(RBp3r*-sau;e4W-}P3@Kok8+_i3!xd%4iczC&~&7-OU%#y=e z?aGTH4hfWTNuZcp>2KD6ZmOlX&u$tm_j{L&;|xkT<}!*RGL8qh{@`=j()g3hnKs`z zD}cx+GGoKj^QC=3#!ydw#w+M4(6nC5qfyTRK`z}Gee|?u^Ne20NhX(7t%YL*^8OPm zZ5Q~=q5rLI4p4Rfzq6Iq|88$C<9|QT^N^4KVpjs!Ru#UWi~j{{l4r96unDh!G_wHf z1^(=_RCIw~y!L8e=a)3md|yL|M}n6fo{fZs8jJps`Z zAPJSyJupbn7>L#tI~61U;yG@m)0Udu(-w&FyH+utjfc+E<$WyW_%=9J7#A3kA2O=z zc3n+wk+RvQXdW1P#dbRBd3{X*_T`h|vc*#Dl=J%L0#blxC%s1ZjXf{=+j-Swy(j>s zGhfuK#T&Xzk7f`E=3FS2i9`%GuGqBWxW+B|0MtM$ze6TY(PAmt(aRuY@VV$1G|3o2 ze?}s9B(V)O61D=|FhqC2k1*r%!--;e3#r@AmX%_CJUpvgR1&upK-}<_hbz}6rL=dA zw$4IL30e#-T@uX~E*s3j|JU{cuHgSSd%Z0Gzti7d#{YemXU+6VMe)z$vQ9KrkETX07N)nvtWh5nEzV`RC_R=pZ66nY!vYmA}oC)N*xHzYFXoZVWGjF+>w})Eh5LI~$=$ z9>IGoG&y_AO&f2bHEO0AY8_px&Dni{xrYi7q1WDBrPvTsek zwX~{)uU(EqqBs_VQOO3XEqA3)CLw3B3JBo{VS+(gG2{1Vlv37#^r!9}n|BNv5fZ0& zSaeSsS6XV!`sU2}v)JyRdii2G_nvktgc#%{b8{syQb;{B#U)|MwQDPEg$sZi*Sm#T zcgq|k`XQ9NZ4*>9?_yR&X3U4luDQ1WM3eE2%;u%jZ~JXu&X-n$?6d|PS`T%*v~2D? zFzu>^*IrJ+{8M1Q=KHv0pQRdsn!DuLHl|L9m$Ka-@@XRf39F|{d*AF+rT^dD&E@~v z+3qdnzvp-|Y){5q&wi`_MXldn0OS9{^LOQp)=K9vNX@Z9#!9!!BPO#LOwE95@2H@; znQaB@2Ht7}scy~pkfz2gZQv$E-tFP(#ksQhOGiDM(O(ANxn?vG$qErG4jutP{~u1F zu`D}lZGzGox`AJjEU{T&k5J}-Q?mHVjKUk5EmqDTGuJ%&OkfhZn#D!dNPIR^!^*2l zFASKKZ6+2+)hvCE!@vvJofxBUI3IwZhBao5*|c(&vtpwrxwTnFCMbk^Ac3x5vCNjt z=a+gK@qfc0`Ye*!HMs<+;Qw~}n_2z;)@FZc|M4tOXV zUxsy4U{OF=8pS=CtD6#4m|VpP@*u_#bK>Go2>Ltw4wD2WS0Nf2)TryjX%`b8O$fMi zlZcm%10Up{?$k}Xp6iVu-xd7-ojCCS-r`+MVMH)^kQ7gV#_s1U9Q+R7^m{{vj;6d@_@9M*=SHKGqR&SnU50nXliPl8OQZ1x|`@WYV6ad$w3qD=h3`6Sc@b@4# z@E85v_;xi8qN@-_Lox#W+sro)jpHkZGq9QY#1P}tk zB|%J#*Pq+k2UgX?*eo#)`3J2avuEi38pEOZa~KT7UvV_dz&3T0`=a6c0cHwJ0D$xU z;`j)B>bG})`R|US;Y==4ay5dk4-@bs!UU|25GGnFy3Lq;&~@WLOD(HSoNOv~F~=3t z#CE%Ehq4QUtzJ)?%BP~%F?17eG(*c;1w6UA8U!JMiI_ZNvapURqcDm*sBgOxFvDO9 zyA~+fcA4@c2lPArzvoJ5$o?p|mA=VJYaFQkXWNUVRmIHGp7``y5j@MZSk@Ic#dAH;TGr??SXVzXRgk8 zqz;v>EJQbM*zsIK!qnx#2M+7T$%Kkc+Xt^4H;ykRHy@MG*>jv%nHOxoO#5BM9pP>r z?x`yzWZY$=wN^0&(>;rLEqo)o41vz8zh^MO?onnQ*zNbYEAYg|RE5c>rx%S2aTQX4 z3p$Zgm$*sy75yRqHWpPd0Be%Xq7@+p2>d_L`R>0vYyY+PD}MAae*85aKGO5;aTp99 z<7oI8c<9laB#*=2AMtcZ|NH%M9B)6O;qVa+r;m5=X1u<-ZqOM3;O96<{opf6RI7e8&;8*7<{%rE?@kpKIeTU*)qf3G)}{-4kCWYo7Y@(TZNiYpUIE4Nl7`P#ZEx^OmoXHPI*mFY>c2Sm~LPwB5|1d$ZK2b8j z&7sguVn}!ZvqR+F!eoOVB8U42S08>lIzK-?JYtLG;KvU?pQ_8-{gabcCaiZGyz=BV z!Ylbk2Jlud;spH)Jpz68#>0X78?$^g>ZgzHqB}S7_s@>g24wbG@)PTwC5@+Fsnjp5 zQBow(m&a!Z=8LoALk&%nXdi#XP@HSYgdTyf>9z=>8Qu_)dao^11l?B#uKkh7obY zcL|!r*eLpbf(a|mhrsNK)tR{h-lF6t@PX_5K{Nzp1ObcbKBNtG6u!qnv7@2axyI8X#1$MuVVO9b41*~Doq@oZUOLa`lMrHlXlH$`UyRa`vP^Cq2yG2#fj+&0 zp9zduTd#f*T#{s(WRa{!Jb<$76ec&a94|Y;Fw+hX?*qW6>V^MHf!oE)n0gx$KRM1X zU}|F_8Mn8A3w8-cgo@{t*V4 zI941PyVtTZ(~_1=yJ>7%$q5P1V!oQer(as%GON_|{J){I88tHhue|@?%Ef=)>i3uN zf1c&B-DQZgf3|b|2fjf5^v65hEFy27q>3dtY{?vYj5W%6)!8_&6kmpzUR?Ac7p$`8 zoakzs$<43OBY3aX21aPlea}M^miVu@VQdsVonzX+IWqfVpC!eO^M(quPIn-QOlqxQ*GO zYw6L?EO@ZS+gf849|1uZgl--laO!4v8%Cb4AbqhcV)OmqTmw|a|Mg#I@BcRYn@j!A zvpkv>f{Ek0zQ^^)L8NZ@_)b_ii_o*+-P@~+ql1s<$Cv-U+CO}Me0p`Ze{u2ihx5a$ ztGCA|M~$jf6h1t@*#F_==<4|N`0{xFL{vUFKia=M{&2dUQHF%R8{4li`4@9!Is2nU zP`f`8OKYK{g=U^iY(I2~>*OWsmqh{p+)p$5Uo+uT=Bcv(*y(5O|2LQU|DWl}@PC$k zPV7FM*@Ma0qYKaQ^a2bvzC_xlhKZW=z@>U;>jM2nCKU4Oou1*E`HhO7!%S~fPA6v_ ztT8i8Ye9LtULz6I%&DK09>VmlG~qo#uK$A@x{(JHM~6xOPzF}0m6NcU@9JN)+uCIJ z_4#Jm>-k!~V9M}wW}}(!K>s?EaZjoOA~GnXbqX3l4@ID*<1+!U_F{emq--yjJU=PF zz%lzXKFyeWtUq~c>&dkM6vbt^|S~1 z(mGiAu>wM4?CjF!=yQxPtC~&uT8nh%da99E?GK+xZ3h0&<@($DY+$AQpUeNXwY#;m z5ttd63Et3bvQPGB6m z9&^MsCa$96=0NM)V`i`^Rhb3w?A|Pn=3bp*vuf3TfvlX3NkQS7E!Y7K;VI7&I(u5s z7m)~bnfi12WS>U-KXC~hOhVky2&97l-`U*G#sBUv`Tys6*zF!BEcIg;U;-RWumhY6 zG1nK6IP{&G`84yTymOM|WrCqrD`Sc=ms9!3#l>Fb6#Dh`tXsJo$5@j_Qn+AMz|Sk{s>HQ(sLceZowVipBtbpWYSN zW2l*4{3V#rwPf6IxJN4YdP20FaBf0W2{$Kql@+0p3{Xjd>=DHBfxhUKLb=ftkw^RgfX7 zV1MaZiu3jeUN$(%}ba^EKI7~Y}e zwkZE+tvpdRxO&>zERt1@cXmcF9Cz@jJ3{Ez*#m5e;t8YW`2uYphLFHEj}74L0p|t| zPy*ZRBQtpjP_)5*u|}W|ePYo%);9S!G{|McXj2sohf0Bf69FfMN z8ka0>!iYB^1M_>xhi_Kb?uL+ncAIAsY9|oKD8i7wi7AT=+HIMA=|6k)M-o6xDfH^Y zBnh9u=0E<{2amwLy9KYh{{z5}mzQVV2f_SpKD}BOy?p}CD(%q$Z{7fBv)6L~S9mW; z`AdFgr2maVL08fL?cJ>YZ+B;VN&lba(FNf7W2uxMBXh=jdA{h-@?`q<4jL6!u`nLA zxc{~Vk)APRvL)cKjE8Iwm#lv+4VJ7Gw8mxDzGq@jtI*Tpw-!2Dypf@(#ZMK8S~h?K zd7;al|Nfwd*o^eQQ7E`7`oH-)XaD=Ux4i#(mgn20|6elB+@DA^Q9;>kA<#G+UqAdA zn~z`qs5JO%K6B9j+M&QI`Txx<{omPK`hPsn^M|MZ^TZ?jGaUnSg)3`@&@_k`J#`Ag zh&A(AxrCTFsR8iPZ>8=tBmHmW3Rpq^d%N4&_>ViUcbEA;p5=MS$C0%Y?}c~>F3LS` zX43GDQ1a&^NVDz#g~tEe+V17zf4$yW?*C_b>fZm=i~@6p{JA)-d5FT-V!9gA6se-k zQ>vLLTF!vOoI)v|AE1c9&qSJE8ZT!G(ZPep$xef@JA|Eh5{BFg4QCmBR^Uwg&Kksj&#bHgo}Rjzd6vCsj4(azSSZhm!o(d4&mRRlU0*W9j8k4cEtJZmYWPX(s_*EbFgR$nx$16u%HNM%%zc6)U2@htYqKH zl#QB8WF<9GD!@|9wrZj_lXb=rIm`f-LrjR*UYwz92Z>@j`4%uk~*5U*Xg3r+B z^yaVY9YCoOGY5`2NFas@2!jZATAjm-s|$h>*jfPxXgo#{`03yR_(6hOonb(_?Ef76 zR_EsTq|5#`mS^1-nPE$ka67l?*Fw` zz)x-xpa}-Yhex>8nPLz5u+*)p?*IO7e`hDN|NE~ux0d)t+sA#pm1IH02ZW!{MJ$LX1DCZhV z5;(U8K8d=&tO-L!c?hFq*j;b6R#w0}NWeLa5jc(psMWf@zV=XrQ3zY~SOJt31Fr7_ zQ4`RHu-4KrAfw4mCqP|VzRlOBR(BHnE`hY&R_p2lLhwm_<51DYq?qR;53?Wae^<0n zcbys23!V)4$eS>06sGLHVnYz9kg2f+iY^fd_y?ZWh`!~djOEYf>@VX}!1egPEnZyj z<#8aJNlQs=NG$N-t*n5LQBVZTM1JaWIhK9an)G2DqI(upAP<75k{0JZUb1{m95V?V zqbbBd4)|ZnL5CKJQ3NA`fr~Nn0+&d#o}z|fmvw`640ZUp;$h;lmE}f2`ZEC`L(qfC z1rfXhWA|5-fGO8Hv|9V(1@Az-roW;g1Z!b%3&FJC+3akA_CLUMv(xMJ){SgbE}@_# z5Cj;wQ7ee(NL&)!gb>^$(7mNli1s1~8GvIqnz$iwJti|T@K?Yk0^78Cu(luhV5`&L z?DW?u{sOGv68;>RZ?u2B0h_c~FKuCmw$N(*8%@B-o$_J(K4`ZGDDl|Odo)P^o3IH( zSUNi{N2&-9g9aMnSzCM%e%jeC@2nictxjJD@HMaVmhG2Fc!}KeiAEHWydvMD8w`YN zUHn%v+BJw>DpL@cfIAn11k!!tw+J#ToXheZa6F)+2X}O~B1Ax(z$rbE!+U@sVauhU zuMH3(Fr!|X)!WKNYoS$&XyVZIGX9^)z z&m%a5*h_+#u~F3zEzia0rq5&Rv0eXZQg)W|TFurYxR##E*VW@cos4f_LU%>P9J{K@ zU-v5;Ey5SLj*)*D;AFz4=ZA?ugrqYLqW!5GgzilUuhV|ckPm!S7L0=^7*EE)l|^%K zML{?VWW{i~pFc)!2C_U{?q>$qB(g&G1>j0m%h!588Mx9?_SWZvD?KkSNa7MWyf0{q z?k`ysU{SWLb#VOv9C9B+XAd~%h#lr185`q(r!sgMTwySU5ybc`K{v3pG&m7O0V^x1 zS<(9Ea8Z`&9VE^kcmNKS;3$x~cgY5DsPO0PfvsMz2cCcvL;(rh@DPUX{RQ+;zzxDl0xw4i#3K~?&K~m!yar-pQ%<192p_?@ELv-Xvkvg-n!%M( znim6RE{Kf&y#Ovp**80TVEgsv2HqUa!+Z%y#bSgTSN8p-bfAkqg!kqvdY6URHhqaK%ayUpeWtsPHbS7{~ST;1rRw z1Y#JK5QHkY#D!Gxw$RpS(>x`+i<-sd zuSqf~J+tb-MNiJsHXe1^RKmnU1r@@j5?wR{&VY*^8uj5~DFg19@F_tIFn}z=y|3W< zzBm6o$_Sn%>s}rUlvXgnP{jVWPiWe9Q{W7^w2TGzb{@gL?{fmCr#u)0Fl4e;7Q#8{ zL_QZoA;WhZ@pKdV)G@pjl)t8&#JuZXrcEfBN6Wfb$rh||&2zk4vfC8Rqp$+gEi_69 z80Cjb`QUnRaa~9$+i@_#1dXNEq_XwJz~xoZnnxZqm#!~SR?X(|nH{c0%%hnc$I{#* z?OsGP)mTP?SbiFCBkF>+U?8uE=&a5FULjn1@*7Q%W`XO_fWmihe0Ywz`y=#aTp0@D zjRmfz$E0~ara5kk*1eoISyV%yMGdj2bK8~+=i2B;s5B`7m7JnIxz4O}C*!qURGkx# z;D8&Dx0EMR7yJdQ7f z$$l8P7*^=B9_jkQ%>5P1GcZQ}D-61jcwO2KHp|borh;fl7i)c95QAf0rKHlK%ZaZm%!DXLs_cBby>#O zr*JVw9u3ODqHvut`+~;68E~=vRIX&p;fvN`HlGD>eR7yM-iDvuaU9a~&2vL%5u20} zw=q0+;GA?*H}I{Xo2STE_pdBGAMyo<%=Eyy@^itZ zS?yM>uS-?54s>&_FKM(~<5F&4T01MHeFa?e;EU0_<0v2pieW3}7XqU3b@CQo@LE#U zMd4f&TvagfelBt>XBlX9c3y3`e&PnYw&osz_{gm(giOzjk)aQQ$igYlxW-brPO6wI z3x7q|7PGN&qhJ6tO!5L`dv65&NyyW2l=BO^uI=`F3{+LH#%(Cb$c>~}Hk0(M!DR_D zTeR*!VwevpO4k-9^mz1L;u_`W;@`7D+)MbrOUdu0RlwSHJoT>W(S5B&roUciaIsI# zpB!8`_`M9Si{N(?E`>8Mbc1oO{B^(gE?CGs>cR1|4f1)^ji}CkI$$gwZ-LLFMfcWt zo#pYB34VKR!j+rHDtyglRA3M3n{V$CoN4Py3qxFdhy3Xry8h2eK;VbSgS81+9aKOJ;(<5(EJ@Z{#KP^^9gJh7<@@Z<*gL87i% za@%0eO__r$h&Y>UCH#-z*d(a|W@dGw>;@I{_-TarY0k|Hsi-Q0RD&RDk2xbw5D3O` zf~IU-1C)%Z)HcTHtfm#! zaG7Qna|#AFxU8lX)o__+7V{cY7;u%S6Y_9Tc`2t!0Dl8QmoW25(Nr+l57`Y2ftpCb z+y@HE&qcvtG_858h=UbI(;z`yDxbPZz?J>6i!nERmWE^V3dGTrIi9{vP>#3F!exbM zkf1SeQ&jSG&;b{a-*@ew9RnYE7@!DzI-`%q~bw6$O#2Bk3h zVU*v8oQONz+#vAR8pWqy%u%Rl_a&BEmQkeDUcx}mVdTT4YF#~o3m9SM2UBa~yOGcJ z1k4jfWd8wcveYO+li>(_B5_#cl!fjH5}t=wB#7~q8~Ilx3A|g}U76q@8n(#@wn-ee z2a||N4(sc6>5UjX1zp@op9Ilu*)_?p&?6Re8cc_0#Ykrs^R`r{#V3Tj^%t)$?M_;^ zxnl+*Y`-&B9jo$l>2eudKWX6xW`Rq{HeU#?awVVv*E<_rIT?B`-S>nbHcm6<)g`)x zQHJ?`FZ83j1)B(^1_&)L4e&IbZU?+~Bj2Tb?*<44QnAV}v1Op1%pzoPaT0SN6SmZ_ z=j5?&gomL7L1lc=@imVf-!v5SwJO8hzm2$;NcX3b4Z*VE)_5`= zlyu+~25}OM-Q>R10%2abY~HX1aIvDgo!(MmAPNo04esr z=z#q&M0Xg#fN4(M2)Ie&-cxblfyXOKojsnwm3A(#}H^)1ALRz-&_o96}CNd z>g6g8A=7>|=ftAhBonTAHhrZbq!q3jMv?Pg;20@kOBEAm$ghK_g~3(Fyg|ZM(|U9v zaGh285s7|u1BX<8Nu7$M>5PJ_p~s{FR~QVSckhMpM}!ImdLUFYq9_>AveMk>g2?D3 zaRM(0iovHNZkbuC5hO0Y#a+3nVLL{Cnw_}o65@KJ_Q;L=5GHK}WgAXmM7k?J90U=q zgbBO&X}gi%juVIpie)XlE)g`BWeOJ)%PxZ?f~19!?E=VF{`$9xdtZJbWCfwn`!)|k zx%cfOxV{bUf?*Q)U3QT2+Rfl8@w&~zW8Uys;pulacRS5py)1B*IMQa}s&b^A^K3ET zIxF#`&B9gXNIU19kA$nzcr=A8jbB;X`eL?RWx#+Gt{PDS76%vS_&!F#Zxi@72wChM z19y+$g0i@b8M5pz@bUEc|9m`RdF@a4-ydCAHKHu24u*tEVnR>E)Lo*DyC4jO_8UwX zD2yZHhV$Gn)}OElFGLuh-NJj!89Zr8wu%!w!lst*Y|OkM=--(EjZQ~ z1d%#BA592t&@eV2ZwkgZ2_`t&hu&ah9^VqZ>qJgtqy zB`K;5L>Cxz#Gt%;M2^Z0NoXns$%RB4O(`_Ib8`6V6?{byc?QCAR^Rb!7>>KJcS*~5 z`kIwf;qCh?g1@LiE6mlFu#&~4JDfV)P)6iewYt9^zI*nvjnqUCs-GDn{5Ecf(^oHE z91~t*JBamElX8b2mM9}KWh{qPIoMGYRb&ay7kWMGoRhf_8VdST+*Z{sR+zSlv5Z_{c#%v7c3Pt`Q_wQof!Y5uFr44}+-v#qF%aUC1tlY+g zWEbL`5js|+hSjB0d%L-x86 zE6;g;0u5@19|$Ch<6IH}9u~k18xj5F#sBr;my=gO9@XEkU%Y$y_W1%Uq@&(lAn=Tr zhwx|JQw;AUaI=Els>DvrGN9>>UU903G`NdcPxuHE;BpD+09gWt7&}u)w^CGW+r~uG zY;Jl*mOt@*y-Ge~(TYicHKJ*Ss97EwP}d9&YX&ma26mdr1-ha2t~pPkx&Bp0mfUq7 zT&K^Bj{2+o-l)My$nMlbERzgs?%He6;5sQGZ_#z9`oDynLdaSJTs`tF+Lo!}D1 zgxlr3B1ee0K(A`KMx)LtP zZ}1DptZ@y~T*%pND0UqX@QzXt|8jI2CGASMYOx$w#O&W?*>}9&$(a%X{*3d8 z8%h3Dh?$ajZd(w zpD`s?z>ejaFHV$#2%YIq-!QV|nc3ffDDWxenhm}~jNp1Gvy(9iV>A?sr`fiYAU4Td zWHr(asthiGDOuEq{+i-OD45Nuyyi3)O{!D)znk>$LHlo~0a%Pj$UWUYzlS3! z(yG*TKNJK|9R!<0YKf05R?q9Qs1ijdqSOMcK6!V6uTzR{iP4RP`}|_G17U$6?W4R) zm|k0&K7DcW=J554ZTNP4czXKl+mq+p_abT6(B7d*`TVLBUwZ8lo$Io*$JpJU_-sAe z?xH339pmmrRqVrc3w^N<&suDZX?|WN~P>i z>|aB#T!-kdpj`GKUU)5f$M6d(nSD5T#>?kTH|S-WW)IU#w9Rnf^ks#S?D1U`(LRZ} zsW#gB9WU}1T<=zM?#s8<2`_@HTQpyBt2ubqxYJDI8y&$0fNi;aof~Z}FmHOFImo&t zqr2T_4rKo}+Y}~LNpAi7-E0n{-HN%t?9JvT*3IRA{+rE*s7u^y+-#=#7rWOGfLP}! z?7rE&1KBC>eXg6$!J%%WH|=nhWjC9}^yj>vzW_E?B~{lG9X1 zYa6B@Q?zyvLer>w&t)+Ac;R0=3TCLd$T~3VU#^v9`B^tG>1f z*Qyo9EmsSojg|-3UXerde0fYQ1&7qcco=foYRVV45Wpx6(J4xQ-s05l`V{C@QPMwz<_!n|_+5Sd5nq%xMVXOtp>l5)GtwkG0BR-K3r%ZPFAFX>&5}`42y~-1Yj5ea zdO$*8r*iw<#`=e3R4KLz#S6R|VezRYWC^@trmwS)*x}TUP*gSQ*af-FU#u=6pm)Ks zNKUmsye>nr38GDHLv1VMWhD$RNzRHHmzaE-N^;3kjMDCQxTwMPi)4p+ept*FWq4Lx zH&);^?FjM)a9TQmwPkc$bKhF-1>)cW(}h~>lPX;l*HANL*n$vVLSWc|e*5>oEkyp( zPXjJ*gLi?q5-$IM+#N1m0eKBJj64VNz}8Gg{fV|pCS)}Cb6sdjrIwj*T`M~>R>fqU zxJM5z>iKjQ6R8NW0o8CN*#1tLztlDGQ$c6!QWWRQa~J;VazJXszIr(<7-kk~5a>Hn z)5a#Sk@!T{lFF19jCET@@+j>lmm%?**PaJ7=OcEPnQVE2v$*q(7yz<9Bq>22Nd(Po8khM)#q z3SXONCoE-DvdG!7medp`4TZpKk*WQ=gdf7p`nvfHo1+tWUWl1ulqyL?&I+m?+{N9y z3$Cnbob%wa6Gp2*14J%Syi8N1y7!}SGH^kq$_v_Qczr1@$r#L2S((nsu|t!#V#>In zHZI_bob4`8`Faz$78g3ZBFDx*(I0+s@8oPSV`ahP8}ezLaLIVex>ZhBI$YDTT$0Z3MsSs>&SKTEzuJ|lMyZI;tro+W&MaK_prA8wEmIu6a;057rLWod zjwSLo-dlG*QTMtItljIV_Rc##W1=b>oP_>Wn-4vUHSRYq6~;`FX9CrMZ@RS>!lh)h z5lea25opQ2Bs`I1#KxjvcE3di!nLzZb-{$Y_@o0eu!p;6Ye2@c36ofl z?&fe63=Og`xm3JLvxtC?t2fnb#GqJ%NE32`kwBOgL6CweHn7(IZ1)b8x^+slEMWy( zb_F>v8!T|3UJDETo_|yR=c-cdjZRnIR2^V0ODJ>Igzadplrk&9^jO;^5Ow93SqRbc z3^OYs(i)?I{x__TvINZTv@;99T$OlcC74Pw>arTxCX2xA&O);Q%vE`4R)V>T@m&aJ zX9ADShKT z3Qis?nzX^xjOJ-q(YY&E<)9kbFifLNFsfRO>6oVJh{osOBD);STb%S|`9iO1iBP)s zX=kR;6{xz^5n3h@UXjcSfej#aW(Xp+KzPnrzBZVvB*hLeGge-T;vBiw2nti_^4w>R zZc_5bZw4;1P!Nx2BT;1uDGNFt^BB1^k)`OcYYZ}8sR?;Yy4eBd3k~HOV2Y}&!&!YL9>Ba^rQtmK7iW%=N|_Bz#Hr@y}dv zynYncQt9|ftLjI@ZrIgHW^M*)=ke))AJPrI_z3b`^LD%<&vju-@OmO~%54g7z;Gg%Q zY*S<@q_Jf;Soa?5fS(&;-`R=BH~V`CqAuV)qI>Hy?fE$D!W?w4JdnL>WVUkP+U{FC zvf>9?6Tc#dUZDhN0!m}Z;AG7b2tqgBHPL}_adK)8{7B>TOIjq)#4M*JAMvzKP3(+E zx*>`EmI@fBkkCHnxkxs2WD4Ntf)x^MC`^Tu0pI2e4piLY--qaW?T8!*5Jb#^|;YV<(? zI)u2MKu}Mm#$^t7A!wNvpOQpvA8&76h)yjpvwW z>V~01YP}Sh>Y`rmsntqnqlkjQ0HbzmSE)7CSuBcz$y}hnZGE(%TfWxY3QHl;B}cFn z6yOb@ItKg+5WoG55jizdib<0W? z139GyOBOY50^s9eAqr~6^N03pX!Q3`i+=H3)&>wZorcAFhL?hznZjUak@47eRj&Pt z>Sx=*nHlYDo1DdxpFxS3Gh6cP3^tEI7tMLD*n^)$L1?h3M;rX=2S8!2yJ6I#Yg{9tPy;v*F8qy<-=)Zg7A;1TL+!qw%@4YiE2Veak9JjaKHXg z-yOXn*|k4k!(VYFOEGKDwQu7osZ|J2gw};$`zw|30$889{~YRax<|Ti)r)^Z22CI$ zg&Jt=Tx3F7QbWfTcqeQ;Bu43PE}~e>{QO9Jy7eSu;4w|xD+@QSL0myEqX{pk)u@uJh()H(KA=&3A2>7TU`A!h3Lm?T<0SA7*3v9%8&hk@d0NCtVm%Bsl7|2lp9M(?0>4rkp9pW&vN z+NM8J$qx1j%VLqR1o!GDizX5InH2>uw}L~Gu}hNEg3egU3NodVj2O%6UZPcz0@aAA zRHTORF`0|X(Lgdf*w>iS$rsok6-}iLY)1bL(-q6gV(!38SS&CKNP@EhXNmQmFl-!v zUJ8amKz+dGkDrHh@Ktv?FNKKr5g+w9>kYLLJ;U54v&v8 z=aBXl)M*428lh{U_-Kd#il=Nwhva*+&*q{kCeR74^?(jO<%Rg2#bu~=^Yvg<&@7%Z zIrxNsg*>?)K>veJ>L2*7(f)&Gk?UQk%@NqPq>|`3tf1Q%Y-q<}UKsyiC_M}~nUmMQ zoW4_=vS4{iV^ki0yZ4h`eSZ`kQGrKen^;)Y?|4L@V+}=peG*>m4iaAMxuX-9U4k2l z%cUq&1An{slN#FLOCL-`l^WvPy`R(|>K`+VMI4+^Q#_=ei7>#8#=P^fG_^Oz@@=c* ztNZf`^-FCka1Ps=LmmWdUA@n5zbCVwYB@9&u7Y|`YR#&mK(9LKjyV z5SDbEpJ3tnAuG>MGbLd60SfQ{8Bo2`L)JJwiZ{?PbP@SQ>dXL=5w&x0tnI)g#WKDk zeg-I=dBL2iCK zJk!~$ey`+$SG=S&O2X5rV!oFbG2A{sMBiH_;t!F$h$Fq85ion}Cr4A_*{QcI-d2J9 z!9Jxzy%^qmz!QyFOckB+5oT^%y;VtGK)QQ4a3tmDF4F|wye5|eIQKX>6pEp2Da{pe zDL5_Np0(DMQji(Vz`;#zFpt!@*dLlCN^}vjoD^zPy2NQf0neKuN5kbb34!b&NRARY z@=&Km?bMb-vY{Bo4brzFKHmRhQ4Gn#Qx;%l*XMftd=e(3NP(0ADCv2fG^LK$ zbCOR;GV*?;<7w6Fen^z^&v zk^1A{>37k=HZ*;@yW{ny#*{HlSu%3UIQ;a<_ah459H4U&8NI%%R2P1 zrOs%hCv$KoYxIOSE>R~VQk>k!xe_Q*(k7dyD3XhU{8)%f>GJFUaKHWN1A4dQ9i8XC_Kqc2LF2R`!#L2iiHxY^F_g{L4dI8ws+@L= z=o;Irw)xgC-LQIZ8Z0#5JZ@vB6Jp>V{q$E;MiW+;8$>;R=AO2Co6+K2Go7$c{3G+4 zegw%Thiy60uX6d^SKLBhGn)T^K2{QSN+w}VEV2s*HFipY>Q_(#lty$>Kf(3kDSR7O zjX7DC9Q1ah-JW}1rV`f2SfA=Y|ElJH_>2{0t%0`|0F@a1t`4EjtXrQ0%i?0513QZ^ zFS=37RGOiBQ55xR55U&G^x@ywy#DN>&RE>-j_AB08UmM1&$mq6>c}tqZzgOtGu3`k zQOi$92@+7o(xoyN0BfK!LH))l3)PUp*_5R-6Kr~hN;z8AvO>z{QmSzvNonc&zYm!P zisCJpW<$maw2VWXj04FEKZ~H#O1Zs^d1{$RMC@#{7~->>nF6eYK2}156`zx`tL0l! zyWR?sh9I*dQyQPc_lhS&yX4rUPZ%$!T7VG6MoIVlmXD!kw(erIFK{CVT2V>Q9 z)K;=A`B1PK&k`l5*xz-)aXcTAUA%7fo%TVkMtZ?M%$SsP!hBY_50NQnr&X?Nx1DMi zF-ZPli%MC?F{v9OfnzkD>TWu=9k6#AWPE*SO-nJ)6^$N}jsO1d|2Co#Br6<}jgR8v z$oy92O4^6;rP_^CGXlW^fLlg@dd3eoUC54w^Ve!=4`wmVoo`;;?YtCB5mD#cw(NFj z7G3_bI_piBh`sLC4&$~qR^nkAT3w@6U3g~o^ zaRMH;4HNKlG+(OM!%C)(+Yg%(v-%)JwoC7cXXbb(v~{Ug`_tR?=mb(;3L;CEt5)ia zZ-W*+V|ZM254Gd1w~ZksZkMQD1AW*tSNg49`r(M- zf@QQ6-1lr;pzrNiUETS*i;T@7SE5tiw$>Rg-`#_Qz5T~ecEW>42m9gvqa+TW>^+LZ zC!>TV`!xA(FL~%LRCNkt;AIf=#NF z7U>+IK+h3FZ_dp4XXX@ZN7qJjRYTtOQ*dC7kn#gPn-p~9b{F`yX?9xOW{^yRLn=pu zFj|sP#nS{$LdItpBp5|jBj3SIof*%)O)f=yefv8D)crfYnIYAlDgcl`Z@(Fx?c!z( zVd0$8Q~PPzn+2KWcir+BS}r;8NT^N~*KUMU`>bFw;}@E9cb5au2+jM2rj6I4EsL2?Q}{5S zKKYf6rb3*PUrt`>WP4+-lSMV!qLX*8qX$b>n2c%8NFpi-X{pqy9~BrYq0d;Er(|4Z zAXY_^KtZaor_>+3UeeZA4_fR}1%7x#cpnBMz%c~0?^ zW2G3L@{((a@vyhaWSvK%g<>`ykfSVhx$nV6OMC2iJfeGRtQH%9x0bkIg%ZK=E`-*w>=`;OiZS7OLv42+DbTOcLJeS-{%4oN*t>`H zy1;E%pmQNt8#7TnqV!}Nvo{bAWxYR#gq^lNYQ;hAxAgeXduMC66K z#{l&(eU>Gp%F&QtLyi%+Yw-`Y1E*zGn|fUf{69Ams{1QMB>yL~3LE33&ZPlQ$-0+D zVQB~yIg6lReDpIb&p;}f(kw}#0)S#{Wi=jiL)~UVmauXXz2;xy4Gw;P7BsCZhJ%|I zY|KBBO|!qZg5W44qDV05a`%h+9TrWZZFDmn4hCKjjOpx$t!>P@U32Lyh2p}D_axk7 zkz*;A#NT+1@py;SGqmV8BS<7MB)x3HvyXUHz@H|DEvmm(FxdYgCI=Xj!8f}r->D=E z3pQaNhoq-~>jgIK#}bPFxfwm!YDNv6KanU+7X}7_9N4gSL$XjcMYxH5zcXTpg#g4TJ1fp z%4q}{fq48usBgcy?Mt4T{flv_AGrGIn-3q4NeH)gg9b=eB#@j{!y`pLz12K zkbDn||3h>3?g7l^D2vl7VTNq4w?q7Sk=7f&!g}E(LfAK2x2}G8vtrUHi~}S18a#=1 z_V>}rzzC0<2w=*}=P9_p|T#ZsLr#R^oHXb%aIeLB(#sxFsf51Acl{N_q z$Z=Xt!lO*DaD&xkrncYvg3Uz9Y6)9-HtAfy<@pB(qg0GcWWmXc!{@JGM6;w{R9VQg z5VI+Ii(1R3(B=W`E^o}LmaHHMew;i1psxHbiB{P_v@>mnh$CACc|X7(LE50^TKbZ% zoFInjLm*%B$&?&kP@Ym9ovrS)Dg*B%E6)W+Id(~*P8cbw%;O>nB~#T8_4_e)AYXBc zmHF6le~slikTol*4ZHLP?C`ce=v0$*U7rqJZ*SnAe+aa$(;5S7X5tb=dJ znG(Pu-QD%|*%@AFrB{UyKm>)GOGgzgyyh|dQZL^JU-i{yRSB8W4>@Cn91;gZXr?y& zZs)sw(=3HJ*$*^Hfbk{Yd^&#n{KL`lb*&XmF4TaMEo`Vb63w|*Cs0f`J zd;oMwc0@II2Y9X?V`pb)=U{&y{@vNxY5u#jv$OXfyZev#4|aF=cMkUdV`p!F_wnQZ zAUmG}0hKIi@gF;P9;-RIzsTp)^_Pa=>*q^;7N7qqEfSWowdAoq9GwN@a8b&m7L~<_}G-TX`Z{UJJD{mAL>{3f`rMq;9#Cq0=bgMazg!7 zo0`KquH^K%5FfR(KB#2LbIN8SJ7v(yw6J2gi~@z@zk4V)7~TjF^sdvYg?xw{N3&L7 zS7|5Oi*|w=pMQ^|Tz`)(KmWUrcK07P&%bK^_56Q^PoF&ZL)TD|-8=_T;O7zf$KmT& zAsQ4)ouENAc+TRK7Hm;%GM*XLvooy*fg2vBU9bMvNN9nV9fI zEo^Q`n?r4e&YMA>&}?p9MVj-KMzsc}u5&h*Cc*5C<_P!OOpT42I_Pw}R?^o-b>ifF z1YizejfR9}*1b-9H+pg}yTmtf$RjUee`cIe3kVDmRaUZsC1f+HPyoMRMM|-1uby53 zzL;CvdLH<`8QbIv;o6vlBfb%!`s$NV&9|sBG+m<%ZVu%(d8xkQ(E>*qDItlK8pm5s zMG7u z6o;|5+Y9<8bq-7sbd6X&=~xuyknHa4J=)U`^rdRr5xx2C*VcJ*4|s+~oYQlzSOGqs zx7$>b)oYsjp;G3Aw(6n5u?9MezOEu;U=yKWqa4BJbUwD5JYo@HWt?mUw)K$o?BBL8 zuyQu4@|(I+(7RD#jZB52B&cC-@A1R$e0s)ArEy)~2Yb<=N!zz2$zEr+R8&we5!7XG z^hua`tq})SOZU?NYD6WbB~s5`mp zRt@O1qT6_({Zgblu%w_f8GXRYV0LaEMX^)CRByP+0b?2~sCmy}wKTRn7YwmCDdf#wOM5`eNyK~LRXT#vhD+xziY6F=)Vqu7eMRvKDgkNynOfW_%yKh z6nfn~7y4G#Gc*a9p%Fw2Vq~}n*#GQe7MJ&Cr>L$6h9^diKtzEfVV_VSAHc*ahV{-@ zt&8;?H)oXM#i@N>681->p4Y1UX9bBXOk+xd}OEv|z$MWzJOEvHLpOAL}f>7qsy0#Er60dRg_V4ge9o)!5N496WPz0ACV6r#f-XQViW{ zI%X&9V2Tw>fZC!SLgu`D<949G`kX^xc@5?2W6HnXi~yLt1E;62Y#}NX3-Qb?X1g`@ zvQDldLpqL^NyZcjq{SSIn6d(0AeD6L3c#8psNJ{5qRKFFf@=Sq7uM&wPkz##o~R~4 zw?}m`>fO;~UEn}Rx*KFgoKK^IKtW|Zv+a;@Ud)*1Y+m~w(`U^leWMF_n`8VyTkoO+ zdb9_M1|;W7sD`jYbBW`Y!XO?KAPKp|J<1COFlpjD_-eAN=V{5RCdX<$qQGoo%s2we z0QV{Im3&Oj>PTzEdP4iNPP{@BGp32O_3?qMnw1C5{rT&ZVd`YJ7Y%EOejZJ3E!0ut5w5Q2$E23P+#k*Q`r7-l zaUv{hbpAw=!{Z}8)LXS~mMtHX&XvOq=CvowZcYreG4^{0Fh)%&$v?pTQ>)CPuxebr z%`*3icMf#P2AZbe!GrlbfeAIU_B8KVfPOxwK95g`SMLM>A;c6KF++mxQu~eCRJ+!k z;5cq?!Ad%5*$~|GSj=WLONQiqukmsZ5D|-fuIK<3cV9fRN$41+d{oe49*!zGAAw~- zg8=54-M|VgHn14N7IbRKrastm_L!_v?;3T^wJ(s}K{PIeln(z!ENt&pQyn?ODkC+; zpAl1};yUeufftHD=&^v5ms%JsD;-XSR{pknkJo-pZBwdft~on`Q;cb)m+be^#(GEx(TzH~oQOV*46-(nQ+-rQWr8`28~ z&}FeqPDP3LJoKz?c^C|sBX$2tKGNk6s(eyl>PAm7ta!RY{fK3tRYwJ%Ov{epB8bQ| z5OpfV_4$D2L$VVDAiu2;+skPIX<<-&Z(Igi`gYsIQf!kw3uSkm#%e9%rJHq-FE~8L zJ&h72Af`r4_AQKSQm`A4ESC$clVX|2T?7lB^?v)8bbI>x(VrXIf359q@L|f*oE0)E z^Hrwc753l#gS|%$`|qQjokw5ozn|mNCnwsmBKY(vGy*^mEsrMe&IePuy+^|9YiQ^x zYSFT3oH{yfmiD#0;8{5)y>I29cdHIqmbCs3`s7!hCE`*Nox(>=X7C^|zgi8WBH1)a zoYEBRq3RL>ClusSwgc4!>oeic9{p5ASl#KVptUjx@suT1YRnNrGQY>*!9aoOBN(2B z>S^fpj`$6=aP7OvC{a&WhQN2#4{SVUaXBQtH{w(Spnf)5i6QB|_@km}ua2z+U>MDI zZH>{|2=%ZEMta}xM!WmrPLD*-XiiO_Z1gv0t&4u;O7yz*!(6 z7##&m$a&RLXPAcVVYarqE`!#K*v2FLy~SG%%omezPW=p?){i3ilATzn>HO)H}71ekaf2Sh_?H3sxb*_3Um~BM1>` zUo_Be>FHf#u~8oSEZ9PjZi1}ZvAGkj>W&x-u8+F^#{%SiD?=a`HH!Q>(AR*@$744R z7tiS~xLgZ3>jmOn9!OA6Ucnu*h(^MF6FHoGQ%SVo`aDm!YS+FZS?L*l7r)pv54#@P zmYv1DbhPz>U#weR*yGZ-yjgaV%thE;MQ9`9AsrQ^Ck*x&cx4WsqTiEg;hdkmX)M|V zq(@<1Z8|5RsnS4sPdod$Gn&94nWMc4fb~9@=)n&kdRuFCgYH?-O^@{S^A9{(4+stM3|Ne{ zk$!5Z3^(3AX}{;2tXS>6yR2EQqwZD1`(V8uZm@DS z#N2M}{Gr>crz<;bLgm_yZaT0MkzT7=k8F0l+c{kGS_!)bK!3>|N(QpmygW@;_EPhH z+?-PRH#P@y1m2fjE4w+-1k1GOeK14nt$_n?)ufhAl7~HzP1x{@y4mVLeH|?x*kvlw zxo8>WexqLd?(B75zN;qVIyLiIv*)h)`KP{qF?abrMGYUJrT0vuZSPJ4bvP&8>uk^i zXx;f>>)v9%9XC}Y}hO3`3;w)4wicV2E%!7Om^!cNnVJ@-Z`7! z1wwDg=$y?-&p1cjSEr`G)ikJ%4m1DzSb}cy-#y;H^%|@o;NG95_y?V(9jvsuygsA- z=hpVWhF^*dixG{x14CEDe?NNssA>Oue6aI1|HEhbz<#v4i6yp^Xo?fGlUHhv@xV-bGOr&?S?f4^ygrr||SBjMB1Y~BfEty3?Nc&9P7-m^PV3*!~Ch?%Z8!&U{O)*i+ z#@|Fo%V5*Bd|n>J8aQ`apqm19*CkbxT5ufJJABJBL!xVcXYfSbm~&&CEzxp*5j3t$ zRNI(#bi<9&pdvT97yJdnOJQ)aJ7Og@i*+oLQ+*@oWO{$89V~MO{_T0eFL=r(zWb%; z4QpL3)o(|~&+3hikDj|9iYhyluG%HkbRioj>es_;zR}Er(0YEy>&^r;G*(xSx#HzU zOok?Pt;mE=>%D$uEN$;*upV^-m_unwHaJ$iWS(=d6)$7)xx-r7#K2Z9I}m195u#3ImZa zc18y&zI^xYxWhA|6YfTe(^Vp;_hSeDe{b^NlAJ)R^8f7Y?lk1TgM)*IU**5g@%b|H zAI)>Ac8~meikC$kN)&Kq(7((pXV*1<-Nl&{9*<1ET zsPm&AI<6e=B^$$0^Q^<`#RL^#CTCj4(^t2h>H&6X@Tt4>7D|0?jmRhVy4A=sE`%Kj zGqGIem0EPyLV44^)iG!TZG^O9GQ9b8I^eeEvH*6!+vW0ct>qPI_FV(E7A>*>)|$`q zy5Rb5)JviD8?FGhi?LLnB$LQPTdDUaUdZxSUQS=)E4pD_fO&Is>9w`9G@A$!Hpf+( zp4!~>?v=(vk8iRR>L%T_&|>3vTRnFLBU*?WQ(qVb5CO$+tsC?axp*RSeLY;*tfUhM zuZtkEKRBP0=KH#-|F%|$MZPz#-w~@fr^TdJbTJIae;O$ zsz{%Hvw3{-_P_t};g^$FTebc>cDq)4)yc5yR6h=Bk`$`nwuJXKlaQ3cR$lPKd`HYMw;e&hw1m*LURQ?=tai-WPj%7+zmhC;#Zi{bb*Ubt1D{LV zcN@8VxJ-*sp7T7sfHt=VZY+9K@+ec=^~`y?Z15Xwo^m5+C4Ax2u3GSze8iQ zcqTHa*5~yH>Y6tqQ(}GXGPVyvmqT`Uc6OS0x+j^Q*lHiHC`la=hVH5t!-V_Q6I(@K zYP<0k$IaDL_1gL|zl-e|_D#oH?d~jH>-$Ee;%}cVWFumc&jE+>UO9+*VuRJJM5FyG zo+2c2>Jy;Y`*P*&1Qt&P>77CecPuwoIDL&86ZAPc?}-lT#%yWFaEK#8PZGW@vzmW*V;Whef!JFvlmal*&OpM(OVl} z9*_{cXvPE*V3S@ON^(V{qB5Zp8>WmVB+SU}mN$-t_a?(MCb@bPf37#0PYXwj#_|R$ znJh{8&)TBlpx5L-|5LBSPn7F;PXNQ}^m@?bBQI+;yj_8LzG;r6I`)2icX;yii+BE>jq{F|T5pi6tM&}z=}aVKe}8{z`-Pwz z^l8W&Xa~UUZ{2V=Uum{q1@+?Qi}6ic7;mjE;{0$W*4r<3ytNYZ>%7s)V%5)GC)?3f zp1UkvhyYQbaN_4fG)_~HA$7Ih{3q2>2R~FJKk$^`U!qc%Rs_ znZ`PSZ0w3QkT;Sd8jEtfK08E=Hb!sRb(j3HHY<#Ftqun59xRFmlzX!)?(TIe+HT5M zhj=9~W3QVHI&1gcALm}PDy|Q>F$wz1i`8bojQMX3gc!4?PX9yd;=;P7!LOhH4L*0b z{~~={?ESma|M%hk{(jT{_u<~oSNreh_}tF^>)&CRg$$(mWl^V}!~M4v#JpTB;MDc$qr`LO5i$C4vwFU8gc)WI?;a|YvwF9dA_7+^Roc~JBlV2ViYr9VRKi?R?uv+z)=s=JXw-{W3Rcz!`_3_+~UsbK64~kq2gL%>+QDm z-m?pw-T*5d;5yoUZud6?VWsn1yV(jrZ{WGs$%Xz^9oOpC_`6u05iKjc*8D~{_gTwr zrmwqstaa|GoBJ0u-&@1GV3qLkmm=Dh(H4b~>-6r*y?uqL+ZVgodAT-g9(FpzUq>@@ zUlgJgu}FvH-7|C*!dyYeX23fc=wbv{@qfFI8}T0xclI71 zeC7W>$LDVGAN4eJ?J$s5nz~MQ(Cxh_76lXF;X7#38Iw?Q4RW(09pj%sb{1reTu6Uh z?N0LP)9U`bB##|@=EeBpicGDvD0mnC1DVad`k9? z*)(}ipFs$-t*yOqjHlm1ahrDCF+IXl#g_NTW-0!c6xrh0Y5&jD#t`yAB_*hO2(cWyxxlBG>TGUz^SF0ci6Xrb@z+B&QZXTi$kA zoB86ZbI^?cef;p@*ZfbP<a8<$^NiSU}`l;+-+gYWj;H3jwKSIcw;t+$T8$ZC;Ie}V^@^+ttmN{aPMiKe{%ZgEET2Dv{@=u| z-}eQuh4egnNhRsk?f!$7#UInpLXTt0Q__1kWt#J~CBPEFBn&pC7g!s<)eSj3KC)U= zOK44r>f_U6UGdn~s?;PlFGS2(qNsDPPB@qR{iWCchUY(;vUJ8LnJC!$W2`#=j~+Jj zf9@UZ?tMM~pX1Xf$FwY2ftBrWWl^axs(1=jS#uhn(+QJNpey%EEV3twoU$|}>U^VK zhi8**Qm~Yk{DOg#wD&vBlAupAHo=m*n>i#=VhQ?2|DRhC)|D0+ys1DUIfKG{EDECM zryovBQLvy-oLy}CF1Xq(pyEN1#EvGF!MC6dX?-1)cr-9P*Ah*p! zQv`IYVLu_yP6(HTrcxNarz=iwHIRN{5XB@JFNSR$QeirmU!5R!k0ie#)aUkQLjP04Px(WX$9 zRWW%ZrWXN92C0wxN%)~UE zt%2I^Li{i{t*9V1F@aogN~rNpAv$G=o?^}l&|v6Fx-)u?6)l+rZTFc;7Jas2y6VJ; zs`LV`FmhIsO&+mGmvtv+?c^_KTS4&7+qmi=YPwS}eWnvf@~9G-&Ew{;o2U71Wj55B=~HhsCowKLA66cU3aELR<49diRj1+cA+@KbZ#4#Ma9md;Q#x7 z{@>sS^5DT6QL^EK2f+{Is3tt>+Ovv3p4)CPy$l|*ODYMKQp6l~Cf*grExRdk^byfT3{!4QgU!)3jx0nYSxJlk$eVVkfrj=yyLC%*^&rb>kL|` zebOXihl##5?n1?=>142Fc!UvCgd?#^o`ox+0Gp}poUoGew6-+XYU|eAE{-)j+rsfK z6gXp-qPrz15EQts!WE!h`5lWJ=Qjtu3 zqcu5PJr4y4@~>twvD!EQ;1yu+t8d0y5nPdDGXS|F&zX!14gu%nioEzpXL-uVRd7W@ z^&$G-u>K7ioLP?Jj9d}8wr#Cf!?TVUBsf6smL=p$vroXusXja-x9J7TWM_^IM4lae zPi2kcRW#|6qA^}GA;`*{>5|mn>s(?#48QkA`oVEB3ZlJ-kH0&^DYa^z8=&k1(8TDA z&mGV!a@OmeX;i<}Rfm?M`ng`R^-$gIt91tbsKc+$>OIIKgHb9*gBexiV(?tV6$BP3 zhzB(1(JblL9SXbTm!;y0$A_Ec)$FP-uTVJ4j;N`N$;mF{P>Dy0owo)7yU{h9Ngu79gf+jU60Ax0@tcD5MEhH zXezP^2y}3>0b{kTxUHBe29ZmTAH_WgFXDF6w-DlAFP2pTJkvWB6(MS@fv-8aB4?kj zHO^YV`DeDKNU7;4fLm6wd`xq=7R1=epl)M%2T8{v>Y4fHr@nXn1ceG3M|3ZztYF?r z#I?w)QOf1CsY?mfp5$X5n_Utq)(VFOqF<#YSC@%X3u!D$b?!2e7#2rA_f!MGFe?I* z@)%YdRoYkN?7N2t-!b`|fPIa+k)pw@Uq8q@1FX6NT@RWm$gia?TaN+)V*af{$; zxlG||zZML&3Yzpvtz^J+d8Y40U^maKu0suAgs2)6wIqeA)r$;Sm{t-aMXP;1ho;*X zT+uv}LP{F43xrb70`B^y<2NgnD?ok{zyp6)IF_Z6RWKcmji9NH-FE9lY2{>F4+Gny zp@z{nr_;W8d7{SM3Ls5Ry%8&k=eju{4-3{k z3HgPk`>w;bvBn88eoUvQuWWQTiKi?+U$M<~8afm|odrpA^p4$$X{HDQc)e=60(TeIQ_n8i#=TFiC!JV;`x!tpiv< zrn+hX1GTb0H8H87Z|KQ1T_z*&+c)c{NeIw0P2#KuN_RuM2Qsb=E~u-^%ntX+6EvQp zC!Fz;dDTNof;^&TT2A6dnwyVc@{0C>j+71L*?A)agC+i#2_7$h6P$OXOQ_puPT5Q`sgt@$uaF)70r}w=+?Ox}`;wud{`0uA`!IRGvYZa;Z^QS?`Ee*gMe;{Oj$ z6yT_2Y84Kif#1%uRF9pXRq@{kj}DsgKRdg-U;Y0-#|Ki4=xlcZA+#v@n8rXVxEQ27 zW|?F|a+uS2%EG;9r>jwRE{D2)mMysf1mV;#rBD^=u?w;ja2~qw`fEhXm67c~A98 zQI8dw26O7r_)T@U^Rkj)-`+6`F-cwbi&}FYA}-AtZJ)?`MwPvQ=DGd47wzsuyFuby zf`&u~N04?;>dHe&u4KStAirs@{V-)0EVao^rb5Bhv6jwI(nUXlb2h&eMFKAU_@ny6 z;v@(?GeZy(*IO`C+=NLC7!o$4JROpf7W|Khj|w{Fv;UeY;Yf^21&K`}15I|He7ED? zn$fIc>3Z-GgZVPq?UI@r~msqyN{doANKb4AAP0&pX1ZFUV^x|z~Z%+ zJ__8yJO#;7Cw;~An~;1PlWeLZ5}yMZ!?N=_tEMGmgBdRhQLv9D?}{SNCI*CL=+ICb2tpfNpFKaH#Olv$^0lF*U{eJ4RyrI^u@$DmA=5O$Uj z8W%!J^1X5QGMdgGPGmM#?`U^teh%Bkz_>+njG%FdCQK0%Zy<$eh+TjY!JcndgA^zDEa_gXW zz^Y+osrZ%293$5KF2RWbO`S;;^hv?S4gt$z2QEc%o(h`4Ph}y}lofK|#vJs`pnwaf ztl*OLP6VXb`2}})m zr$lN^iJXd}R6}u*fGUR1n4&2uBO50SZPD19a|@d(K5#;F!ZX;23L?~$%`>gUDJQg;fP1m52VivE9(`YDF(hG#`z)ONljkAJVv#^{f-p?QBpmaU zVJMF}MB4j47dm{D7L&Slo2XrW3pUtyl*Y}M^3YBxnk51ISQhlKm>?Akv`34{H#j6u zcAo42VKW$p5Fy~ykB(u+Y6UJqPvl=c>M{i;^O zj}|FhU>9?#&{J=3)ya~Pqhs0V55>U~Ea`#t*+Q*H^5s$T=^El(JOdu)8-&f4B*q2b zDyW0FlBJm0^rF`QI?eM=UR`x<^rFta97hPuchkTE?jQr6eAkry?Hd&my|yA{|OAn$&ZK6 zJd6?jRI_vN;wes%s-w&>f&t4Ef>Vl`lX^fxBF{N0NZ~tlq`rk@R4LjF*P=iLs_!v} z1_P0@5f+t#c$ae94X1Fd&K9yDm1HmUz$Zc-;<>1b(7+O^w*oU^-)Dz}!c8tr`S#o?GZdDZJCA2%VfWK7+N0JsTDpwcfqr8qD6 zw-iK8nUrEHAgZO;$X|fNBxzZYpTX8XF$=fh>WnE0u6h`Fy;+|EW2WYrqLaT@Sq#2a zu>G)0V){fLg}l~mOI(XCQ()=PE!X3ejC)^i)4N!vmfK6~Fwxh#9z zq<2ZncnXh!7=kq)si6ye(-dEAdF?|_epQTs<|WL|mylGZ(bO|dsXN-3*IvGRckCPj zH!TmRR*j`heV?mMqKDXuG#6m)UyA8b+Z_TvTuuau`x-qH)$ypkP%5awl{^X*oa&H5 zp4AS{$do+5@<=P?D>q}E2v*5bJJIfbgq1m*G__2gQj2tpyI2nnW4CVNGL>P>^C>Gr zS@Dv|j%yg`_Z=?ckwVHp2MtX0D@iBJY%O=s64LKmtD$v{(-9qp*|J`0*c?O)>CA8# zyXjCQv|A9|!#YxJ^H?NX&Dtv1-N_0*wWRVKSW`rEn2RK|JJ|-l!cXj@VpdrhmQxls zpM+ykgw(wnc%?T^yNbRnDz0TqzW{S^YzmmYkTFduY8{=p9DDiQj8vTzB6qbu4qv?r z>ctXu3EMvTRgp4`4KJ9IS6POkJf+Y9E_2I_yU1mVK0GE|oNIY37MYB>sHE;Zo(gp` zjrF>cHkjFDN-5O+3zjb6ytNcrR2f8dWEZ>;8J+_mt|>3elQGGxNiY&4BV8?7!xArbyPBFSXuzxc8dt6XzF}*3?e> zib56>yrp#UHg%D85pjuK{UVi+4a;b}FZr0qa0w_JoU^$M8l|j}5yd;LSxCA}%?2?F z;07rP{T7V^)3HUX)3qD%w#~Gn1~x~o#<+pDNe2-dzOm6Q zGI5b%mDI^G9)6lb!qA6PuM_H08?Bg z)LjCz8%@hu+9fJE#!9V(=E(3McBn3!ufuQl-m1w^wfy>;S7_IXFz3jXqtH!DzYK+G z)hth8e&<=qiVN!AR#!6c44@d$PO_j_3AG<3kSRrlo%^2gZj#zpp&lF^$1E>dqRCtz z`5-rj5`@jr8phht)Qwno8Z@Kggx49hblv--9xt7o5rVsVHyR6mgI#p@pk>qRY0rYj{p?DaS3I{*1W} z1D_B1yRO-cnjc^FV=`&x%l34jTe(fw;)MF7)+QRKkTop~$xo@tpN4*kVSJ2%BsR%1po}bVPrVf@-XM8(6C+>45g-0yvWL$P>&Yc`Vgo^Vv~YNX>^EM zMbERYUx8@2(fW<@VfrVc_FiM7E%eU`Ittp_9ias&s9%eP$)U9Peyy6i{b}6UrdbkK zpEJGX5j|*vIl+CeEjTs8y;}8juZpKHsdieXbXuWuTCH$utDBs%NvoRreey;m%rJrq z&v5TZ$tR%BSqD2xZVORx^U4@=kqv@I*<)?cGT}V6^}YGH_6Dix>Ic{ftWgGe$W8c= z1>jvHxaHo4R%=;?G)ue^$%CL)Qdtx7cOn<5n9PaHfobz&2JQ%wGvVg~{}+~ev=B!N zz0nkS`0x~-J#&wYz2VStA(}(1<=66@1EhBIkgN>arp#3x(qWek6i}h~xq=Hg zCn=TkjW*7KD+mYkSI{SUvx)IuS?`n=RTg?qi2kWhMgs-lb1sVt$jgsaGSSSdPN+{q z$#RM+BNG5qk37<*7F$jLS}7`3aEW0B>?4g!{T4Fdint9#K?*iuA5DD#X^byG37{KA z0~40b@k_zTtSV7u%s$3xCHV!50_^zEyyUD(%c0k*W;KJ%`jvdi89dj`R!zXW6g*E1 zv{XY#PEGJN=0db;qfU5327N;!aqNiv=(O!FU!u#Q2tkCP1OQ3cN>SJg%-bd)_aVgR z+b*cqPQ!+sAwCgAUXVPk3JN;k8I0xTXn@99LMBJhtfu^&IfoUk^PiDTW8fqkPhH~9 zr|-|7$PoGocemZ&*>)5~ziq3d_w+r@^WW6>tqnD^Lz8ZIq% z@X^i{+1#RFRO!T&Q>vyS*##?@$!QGv)cMRFQ&|Hap?dJ!#+KPE$TZbAJ#t%XOm*q@ z^7fQR^48s1pKW9Lu-((_}ATZZX((uj;!_Pwj~e z9SC~Uc|YLJJG`Jg1vOVJQsku;J1Y0R4YUo$V`^Xb%z_2;+yt~~O;kYPwclVAxY2a= zWY3($CmBdBXztR*mDNQ#qq+WCi0pSU!Y_%*;;6q`mI1%pY>V&>^3$mOp5`I}uf~|B zXyXcvtie^q+jjOAJ(uf1n@IFu zIV&Vor7h8n7N?ZYx*zFZMUq&KVFVn5eSm)$XHWA$e^>22FHHRim++5DloWr`h9&(s zXy^17$BmgEL_sGCVv9gs$8ITla5A(kX*|U>3;LLzkY^Lvg!qz9W=XN=2MI&(k#UnC z&4eI_e2QI@GY~b+1aOL@Bdi<*){puF{0MgqrivDP|w9=Ly6PUxP!a zqMfP>OlXZG)hIb|?BMFxA!Bz8wqRn1NkTTY*U}bvNHr34DC#`8wCNe@JxgMur%ehJ z4XFS#Y2^Hd(7C~>NOfvSmsqHkt{bt2#~&Qr%$T+7Nf8$>!#5O=z;}>2JQ|yxT{_R& zrG@W|L^(xMESa%lVy;$$Y_H2^Z`SLt`*z+$AO9^P3Ib8n=C-S+C3hU{eN5XuF=EFe z#=+~O&0B+s+8d;Xwy?$_8~kktea9V`>DffLA9bx8o^^KQGb#Qg9Fq-`@mxk5zUcOq zpx+N#Nxt#*rd!LO=+yFOy0rYc4lRF*JB#nk(r{&IJF?W>Sn5tJEf*HF@@RJQ{E5qt z09i;dqmy;j)S8V(!AVU5!HY9e9scBd$|Jl^{IaK=sTxQK{Z?yCIHWvR->up_bO)79m zidYI_+6HP8ubq2C$IW8wH&l~JazJu-u0e$mhEK-36vDU zz#5i?{eGAwVB^;=MD7&~=Nh&f1}3mWQ4Y*B@{wq3o`(&aXGbIfQEd*fiM?5XX6+dg zXK2>$&W;~AgP)N^bxVt~m3q-%eT7@;Y@z;p2T z<3ob+QJDI%EY;?Qt-vmLtew>XAfBCgL1fM0jr*P3g^30jSsIhj1w+fgj*gXkic;sv zTGE5DihBJd814lzGknZ+=|ho3m8=v>b)J0d)uUUl<|_&RQ+k9v3hv_^%Onn z8COjl_q9+2Up-KitL9g!IyDLFXp+?3D$#+$OPmFHzJfkEWz1YeWW-W&8MIs{`=rBf z5-)?GPpsc0{u1Kr4!RY9H9%Sdkq3+wWrhfV1z`r}Ck?_f*j^~nfy`*e$0j4Wj_;(> z2GL@a)@#$qqjI&Z3?^XpoM*{UTV`L=+!!=jNfTO9W6ew(woM&9>tcjpQ2p3{Q9HAT z@EhsDR_Ym(x%QF8LoNx=e%B1X@5^tSMb0YMLA#420Ot@QOg7?qoDr16COR_%|Lp^FIdJTC<8~pw5R%)WA%f=>zMMiD%iVe z!Q@n=$&fq@2pCX)X4)-YeR;{Kcg;D!C-p2KFd;oFz;2L#tG7FN!4J-Y^S1G7=7uDy`eng5jF;axIk|rI2 zZhgvhQAX`kg(8cLpmv=J!L!n7E5a+dd#JT7VBXimIoF_fUez~UNK^o?r~SO^p@D-& zoiRj?DI^pHN10ZiLS+A^Pc~SB^g65-CZwWAB658lt!i%7OS=8i0zpQM)+K@l{)Fd# zyrM}gB1xW74sJL~l*tQGictlHmly`@>Ny{Rk2IOW-4VI|5-^Wg<{hdk&fEZ_x$hUt zOVCM(0`vZ&6LjI#c*FSh6f1c;Uta2WqSK$6)ro@O)2EQ|u|+~Hs?wxvy}k~DW>~0Q zU^Q*@;(;S!0C{U~^c)?o*d$wFA zOIO^nXz0p^!P(ha0Lg*TQsu|yl3tG(Cv7hb*H}shH`|*%>mamI?EDr zef{-z|4V#sdH;WyvNTuhd6}n3Hb*BS0ih-F{P3!ar@()(5pdbjF8(U#`lHOSpQZa__lYW9uV z!uAOnJ|&yRRb=s`qb;uoK5lk`&+utk@Y!iK9`lc+H|TY@1lS=zh4*3u8MWk7{(D{X z?~`A7mMDSx6qV5mC4&cn`PG!Y1`RJw66an6$*9Moz+{(bvak?CDGqw0O#f1h5Vp;l;);mELoLC|bjF3F}& zt^fSZscH*86gi8xqQKelu@mcyhH0-cR=VNm1_oILyoPoiEIUrCgU*_V7P+R)!Uk($ z-|Ro2jxdwq)AbkFgVQ1fdx`u?rViHHuQvwJ-5=q%ve)Ph$GypxTY9$4R&1V{c702= z;A^dKJs-fEMJruz7HJn9cU#!yad+2pwtZ_+*D$Xpq8o$s^dG&oI$4O#LtAE{?QDZ$ zqL<7cYP?lx$wSx3bD6_zMzaw(|L7#q>ryc-HrFn6e{0vgHaf59MXkE;^>#tu6}P#+ z_dNOnbszN2LZz*5X6ZOu4%VYH@Ir6lsqzm!{%x+bD=^WiN%a0lkMutDZb9LHHYVl~ zl{_~2*46rv4{xt_mJ>BkBnj&-KM_FEhu5>fuaY=f(Bkg^yvt4_y^hor{mep%Myxv;t+_d{N>kgjhWHT0u zHSZK`z1K+Vmo4I7&Fl}dspOisW5vewW;B0~POZN|qL{I|i#}b$u?>lLD)ntd>NhjA zvga-$wVQa=0=%0tM3za1xxnRXZR|OeebcRx=u~DzTOBkC`itb0de2;^ZSBjqa`>(M zFLm3yR`tGIWP=1(4kWRiUGLd1M?2iR4bi$+_?up(O%Bs!-)=y*Y~y0~L$`rtZ&a-# zq%Dv8?_80LX9>$nvfpC)eV1E3@Dg!6wMv&OQt#4A6*usu2x^HnZ|L+BC)-|A%vwng!kF4BVdEiPtbyY2GmHJRJjFzp60k}wW zneW(x;4cjnFW1o&jHP+nka!^2DoEoIdUJlE&zs?olDD7K>QoS{U6$>Q%TMUDasfRV zJQ;RJr`*^d_Vqvbv+4fNLkhGa9xJK>w$*<!ClY-4Qu-*6{poP0?pkd{=$POcb{qptIm1FD^B z2$C5UV@w3wnws0yM&r^pF1p~+)+5Ig*qaH@J9l9_#=g5cqdQ!`^{V*yc{Y{*IrUV! zo3?_mE%JYGdU{xq|HntC5Ay$B9_z8Lk-J3;map7vtlntUMfoPe%HzGOD0OHmZj#o7J2V)iSzumz}+2tbujCfqmARvFBRu z>&^I+=Mp%NaWsD;rKY%CZHz=h9m)C{YGEK`GU|rR`b77p8ACTS4VTSY__p#{Q%7oq z2uT)(yXJbyQfZeU#Og~kvX0mNY1Mr303c=D)Bpmx0tUoJa?+K1a5Isr zlT%CctUk1)ch(5m_+h0#{({FV$zyyoNrP%w$-tm}u0`9_e65xq11pe6V#>fJIeLZ) zk|#J{qNjtW1GLxRIMp1NgQl~|QgPKfGd-YYcSF6qZcfv#eR~}@x}?gmiZeQhEdG$> zwv-sM--}#gwnrrm1&zb#Zq775d{j z_H^PHJcgh=s3|h_g{4Y66LY)em*Et4wQK1*NQ^ zXWAmy+J8p>Orr$3`)(^{6%k|23tjD=H{yAqsi)GE>&V#QG~-p7E-O~Hf+L92~;i0GpPps4;W z-+e|ji=)DZo{q~iX1g$a^_P1^7DFG8dL4$gn%cEAu4^OH8s)7_lrGpt69al>_O;aF zOb8kDoG)ncYZCpoG2_)!qc(SMFd&_G*Ua4 zsq|E5LpwvaAZAsbeoE_{QbDfm(7FnuokY!QTq7nODX^{tS>ai&;-b=H^0uDt2!-P4 z&v=f#y0^5ixeGhsmgePxtaEO!{BZ^HS}Q#@pdrMJoPnDzowBA^WuYJ=`=Gg|v__TU z%wJ*V^yH=uPHY>-4Rg5-)Qt>~)@w^f3!5$9mEF}f4w3Ftw@G=BXr_Lweb;&oN`!jb z>>@2L%`Olj!LAY1qQ~fngpu4Xy=V#7(XCDmp!8z8v!ck{F0#nVKXo*kWsvGpP{>_# zx@x%&X^%0*#;^z8x)i54Z$zxb+FI3L=`{)tRSH)CM z`?v(=z6P(_m1fQhaOxT~G9Hn%LTkrs4ObdAj40z0Vn%RW%>HHmq`U-nP=~2DUTDe%uJ@+Os&4}ouZa3&q&52yeS?Gx*z5H42{;YJ->I`ja zzp-l+C`vq@F4Mzdm=j8$HRh{q)VUa}MMjI9;Q2E~fKk7%gxsjUsQaeD4|G!r9J$f= z zA0+&rTEef-X=sHr#V+!nuK4yor2f+fssHHw48WMH2){tG+?6H1x zw3mK}ssXh-=8YDe5Y-atHsWa{+wF#`p8eN~sor3d$t%(& z8k^Xrw;A3#*nO3=G1@bM){K4a*i4qSvJHqYPKScq98hbAZR3;m*H)YwuCbsVu8IFnpAGN-@LF@K zK;G(0KyCN`hQmSi{`cT$@Noa%ZLfJ+3>fk^pV%|M>8zy8aIjPaoF*eLU?!Sk*KS>f$fA*zSJo*U%up`Mo}VP3@Nq z|F$5NTeu^%TXA~{Qt>5#zI@sJ!(}npaQ%18{KlWQ{BI|N+Wp^?QIz0#|h~nUe?!%XnWqJLow{mI%kO$mfQ}OX(c9 zLIo$MP zGC@ueI*X1Nt{LKB#lTSGhM|8#Kf0 z%`zj3gI zGPY&@Wf4wEA@Zxkl{2&M-Py%Uv6)l|fO3xHU^cH8<-|S(}VWfrt8% z;o60zpFc%*ktUx7&bi_TTWJ=Kno@ z$p3UdkH!Ae^MCHKANI631G89fMl+?mYQjT?P?#oU5AAi`KS0{EO~i!c_G-z+D;tE` z70eH}u_tUptk-wj3aQpjBDESqz4J;^sVt9fqnJ!dIO5mjoXWYghjzJhYqtaLVHmM~ zas0PcXNO%rY(%`0KuF@uUcS6IH(y-5d|tlD7s(m@l@hU&=8Kq&(C*I*T3oVUUZ=@r znijjT<-UrN#YdwZv^h(b@cSamm|>I@I2JYbtsr;#TbxYGL;M}|W15en5V0XAQ(g`; zO$#(-fP57hI**bODzECy@y8K5IX*r-b`Y9So}KnyJC!$38IgXBV!~@>OY;6MESZOA zlV{WYpJtwx1N=_szr(}RVRip!CgH>We-FG)U6HqO?=x*5k8F^`t@&?DCR z4HUSPMOcj@+#QO|>&`s`Qnemwy{i2WXy^}*{ed@dZJ0pKM41zvj?mj(K25vtjC`M! zHsoVL5|~=quFTyiBZUE!X-7nawMXcd( z+mZjPYe2Ng|AXUN{ima&;e-6YkLTY@{(tSF6Ps)UcP~6~hu+JY5K#EYUKL^sbs@G= z8R9`|Z~xg){%i5l?FArO_5Wc#{$ubE|8*}<$Kd?0UHQR`zyIdPFZUXIVg{L&cT7j< z^Zx+Q#zr8$I#L-vD~dcCFAAbv+`1F}&s}2v_ z!(Ey5AxT# zZV!-oRP9bLngHNx%A#okQGvfL2rbwk>TF+kZy6B8EV=It=N1@L%^j6K zuD34Z9(NzPA8--O+;{R;O&NE?k(SzaXC$o{DJUCYkf+H%({bni#{ya%2?1z*6hchq zX+lZ&nJOKT3_8v-YQw06y8%wB#?iKsDgm#LmCAf_7j98N?$AHmOgbYGMl?uoNxodM zs7%P~V+QgyO$r=mX$YCa100jQGmi3;D4E*OYXZMMq9R(kJM~Pqs>RpauvLAmkKR^9 z!DeCrw1)1&I!*I1O7tz&h6)F|0W#4%D)^E6UDlcgW_=vy3ub91(+^}kOVb-Jlapx##k*D!3_=E0b-Vs3N(JjKge;+KfPr?eDe8_!r=?bby#aD*ct^@G z0PFXgF%6X#Bq?}oP!Qv2ekVli5buuJR*~8SpEL|r;OrOzXFH?B4?pbd%m#NO?CwUO zG~A$9SwL{>I-3S~W2_iGgET2}hE8`CU!Tz>L3w%e6bV%!CLS?e1%~22_ zHtCzY7w^`&a#60^yC`b94%0TVsSaxsOwt@rnSfkJ5fopo+`CQ4>tl9{<0$+VN=Hk_ z8;k_RJuV6y%+@mFx{$dByfc10&F7=`n(yM-Ut7p6FO;=9z zM|CfHuUg)3EsuEV;oeqM&zAhxR)6${VLr-5A5HjCM$IK|o@Lz4>rM=hI==>JAO8l|1t$arz7AN`F}V(s>=VP!_$ZJ-+Or)GCRTS z{wZ!!TSIj!rPk9qDQ09ruYw$BDpy4Lp6xgkelt@*S-#w@=Yz0h{@{zJ7F1--cVTPY zl=*Rqi|c6V)xCPri+ZY$b#kHD;;3ZvBn^#fOY(0kY_oBNwgp^P zSi95**Wp`pv3S^tTf62J9a_)Xvsr?iqlcip-}!Sl{SR{u{f_b92Pf71 z567p+2M_w+eLQv%Dl9MQ09cU%)UfZq*0q{DTs*nswUcJjtedG}#pjLL4(eq-Do{45 z`eCd0aUYdFT2SvbWEgrd*1zg=H~BBVx|94L4u{9p`0ta!@In6H%hPBJ`}4Xzr0)SG z!}vj(la%^FI`3C^24O_}NjwRks)d{^Dm*{xvOxLfVz*0Tf;g5^*8n0wCFP`@@U?Yhqg=4SsK2nbzXUS zm;M#wRU{~PFTIN9k-lfv5v=TzE1X$sNNP2h$yIBwwH$*g!$LQrHS}PiQmw36bMDxd z)Q4U0k|6XP7;B)V<&FIcX<}6vvzcdD)pBm1*eEGTev9K0N%_Wb=5+XSwZGCvSzy;U zyntZYzwx>a2`h}XF@fSxzvdbdGh1#vxEXD!kd^0AswfjxbXR>!I17K2lvmxjxtj&|T zjV$WIJ7HEFlvGmj%WYX~^A9`o!_NGb zcjgBP`ygTe91^yE?D?&|T@O$7+0_4AP<2(;YfA>wcK$c0`+tuf>VMwP^QoTy&l-$i ziwmwE3w8|GwK#(fcKYLahDC-6{Xzo74!Lg2B|3OBB}M&HoI!?$yQe(ub{|P0!fzr| z{tvkO@F2hc%+I#u|EkL%ZSsG3TCM+jdU|^FAph^<`3Caec3tDou?^hej>k4g@Q}>) z;aT_DQ2yVNyoiEIikl`IpzZ$0;HZ}W`|u(E_x(Kj{lAv%+*;nFQm(tbvJ|Ogii-n4Et!u+bL^WE_ z;lCIzyLEwWhTI%~gj&W#Kgw0_bmCMUovrY;NWyuP)Uj0WHo&rk*N!DW4HcT0naY@% z)qRoAxbqzhsi<8E%lV2F)*uny>bc6=XLHYTxwXFF>bRH|S%?dFRdgR1uD zf7a6f{27Vo(KJbOvQ8f@`rqJqcwEu{4h|mjKi|)@gD!AUkUXKNND-e~^kGI4G+sn8 z)CR*dOB? z=doYRGw1PtJ3HuaIFHf=MK7PfppKvADKoSkKMD!%^LBaq&p)D+TxRN9T+U9Oq?13mnH3%7(tRG{#es6z}$CMUl}_zdwzN z*PYB?>f{p03hWz0S9r7zUbXBp{A`+I*l?SM4-Lo`r(FlzJ{I1teIz^R=V+Qk zf<83K(>bCA<8B*H48lq&;W+%R679upMAuGMN20n7y}BGJ-L4V6RG?pJ>Pkgs$y9L@!XU9NXv&w_$rr0 zIMCj0gb_y9<|)}VurK+D=UGg8d}=Vd){@;_Bc9L`I6jymEM{tn=kd166LU{deL}pP z;82E~@KBL53np@1#IFVDWp-6IT?=!#ytxVk;lAG~DLB`_vZ+k7Bn`C+sWCGt{?)I1 z(=T#D`g6>N+Lzz=C6XRiRm^&2{U5#@sQ*W_tNtj@YWwdjoWtVngP`nO@T>B)`F~DN zs_}n=)8Rw>$Gtq2>%svpaGXx%eIRBXMPW#i5ptQ>cCAfdsozSGxBJ60H1rQn{DC(> z-=m>_IAs6awSJn?5sIT^@lpF#uq69g)K6l&hInglryc8Y*cv~3bV&;`w|^+rB`$2` zV#pSB#-vs?=?TuV^3Q=k^p8C8$$=9R8srh|<|A}P3W^pP!deX|swm>%FeH)Yv(AhD$Iu2b$4akb^QDDP4vq!~Y4Ak^7GI9W>hN~_}qJLxuJrYr^vQ(3-qoN>H=Nm$>E zZ7>JA55I@{(%?n`8=OvnNkrmMk|87sAJLcy|Don!N{F!Ac$Y6HXugPxD2s_42V%>I zBw&M(oP=mETo6VV20q4F#t;<4una4nD*p(PC{mY;+U!xsx%fkJXqEs zXG!U*M8ymjjMQUbPO(Wd&WL>CBT@um&rz*M$W{MVu)rOoiB;eiVLStLt#m8D_Zd0{ zavHc~@t?yuq5C-jY-3!{sS>(^d9W_|i;a`?gn(V7< zupmt{oZ~t5-;3*ya%`GJQA0_K8Oa5cR}+X-?k*osE#EArSU&~VuC57I3QoX`1CfA zDIxqg4RAdA^oh4)h(Wl}4!Tmpd!B}1tf@U*5R+|dT!3+Fh;j3x5OYZ6zK%Z3NKQ1w zb3zNc&$<3XLe^Wk&pFI>UR|t5vGm% zEJ4=G1A+*npWeK=xN_7!#WtB8^gI>2vKXQ8sPje)BMwVP?0?#n#nsT%5ZU%X_=;gY zpfu+_ahQ`d4+x-nB`$mGz}x0U+$X_mjf%Xo!u1-G-xlomjA^4#D~g{SfjB^u7*9n% zLfi|`Mal>-bV3s@)IBQI!ts)ERgt1Oz9FRtIh0aTE85&`d*|p_rq?LV1U+ig%x6y9 zHc;h*pGh%GRrKTzx+FNHVlcqQQlV$22-E(Q_gsAfX53_va5sKL*^*S=ly(<^hF)H zzL|N>%tG7;Lb&bJ;lLR3N(=jz^r_ynw^C@M;TQ)Gty3V%1#rw<^*YelTfIcgjv67? zAUS*Zc@~b>696*C}k-mUm#93n9@4^i)d6Fg&$s%S{ zlf0n5+{8IcA*?k95v)sIIFiNO8N(?c)`c9W601TSv-$8H$;ABF7p4LBQ_w4;3A#3k zCV2E(seMUUZ6f-ECbCc7k4X=%swsO=?w^035zwbKUX-fgVjM?wR@K%#v84x^j#is; zrWjdWG6>EBW#?xqOK6q0Vk8x_Mr<``%$QUD#`|FNR&w)ZCW#B#r^tO%tV z-8`Y_?Bb>93gR@CT0TM6nS&WKe_6G2f|96G;*rtVrpE1dG0i(j zTL)ZShhujre$3v9gJv;m&GXJ=Gi!CF>-Y#2X-q_INk9Z4Jb{vdOW7s(E=S_nRfdJA zxJc;5(KyHX(i<=6attR6 z0tGmp89K^fk>*5{6l`Xg)j6s8gOz3vj% z+dqU)SSV`aMS+sEkS#9KkPqai^aHsi`M$Jx7R;oLmrQ@JAXf3@2u}-pF~

awQaN zwkEe}!Op|sElvU^P=T?<(gTw`*Ew2S%a|I#z!}Gcb~@1yn|x_CVSpvzP+?gU40?-n zylZ1C0q;_jEaqd91Dl9~%rY{F@ACke?efqWC`WAnqY>NinFv~xp;M(}nCWzI6(?Rw zahFHaSz+mpjiNkevpU4Z`heymG;kcS-&)M}VwR_i=}egKmCZm{-zA$UiOnNxr0xD> z@MPd1)Jb@WW)|@UiwAbz1=A>*ff!3OSwCUyD8becS#mTD5=;JU6Ik$@`|crn&KWaLf7c(aAwI{{QfBaPn~edmqmZx)gyGaNSQB04_gyd@r^hNaUK&O_NI1VJcVN7;8+6Nq^iuQmC}gfQFFsCc>@=jsRI71JgxKp8Wr+bkHHy}0$1t-K1XYjoC$t0F?Q>MOr{pXZ9q4zY}7b0=C=DnHq`4>SIwfq36h55$Ebk?(_i73_36+G*H4 zrn}Ix6LKmFMB6;LK)rkgdq|Uvg#E@ZY4T`6`3bJB^VnUx>Jv%|?a;UR!|L&H388+n z(+tJZR!6?sx7-MEfJWaCS(7G z`D~pnsg2tQtwUaC#*FXoGG_)L`>gT(pc{lWX3ikYW^<=^Z7o2Xqy^d|^Q>4}z*wl_ z%X;}*GZ*byzO{B&-BvtX2RvTV<(2_;tDC0;r8YI~<=za_{~}8eM-Y8!o7ugo5LmK> z`(Ab_Sc7-mGprV?$m0-X$=n7oJqzhgvS?_l0u`$;*BiO@z0c8B}1N^9b z6D!V6{1x6_#QmcyE^n+~DPVP4jLt4Um9-g4Z#Ek1db;MsQrczZk!MCTeV}GWj@=AW zukOAx!#&vhP=>l@BJ(md;+Z%gnD*`Nf)eA)raCaIy`EKFK(F>-_wa1sE`gCJX(FRs ze!YAJxkvhXq7gB;x9*e_?z^VL1I`|1450L*nc0Hdzpx2m&-3oUj?MKe?vE=st2Cjn zskL$5yt>3{N>r1wuRgG`O`n>i3OaY1pmmYD>y)xVH7tj2i?4wq%vKNyA*`1t%)^|u zX|}@4`Ax`(P$i=YT0LxkO~$B5k#+V3eS-L=(7zpyNrB~LU8Lca*b#WX?mrd>%hCo} z9^FPUnd;$}S~RQ*wPf#JUYwhaE?zz_U*wDAjQ&bVE&>%7F&UxVOZN6GS?*S|Ak==p z<8~)6G*o6+^x5)p9-2qV2o2O#o;m(FLMO+^hsO>=6Uz6PErH7k4u47FCKx9OJsxnU-$BS8~aa|;b{1(3~AQ^)XHn<9xw5uyxE{Hg)MHPmQ(-jD=-MD zHAX&WXcxXp>H~)h^dgwO+H|NVqEd^XrvRx< z6ni!%q$&&}iG}LrUgX&2U9Pg0jCQ-L(cV;Fb*q_Exs#!d#~OfJ8%qQ3I@s0rvC&dh zqg8iep{pgS0iS*E4q3Z^-65@hAqh26ifDm2^BSOq}BUll+)txQ8D|8em06Wz1BEOsEKgH=3)`Y+y;{;GyIdvkDA`3B-A?KYGhIOcWaPsmk||4m7zD@0`NiN z*X9ji_*y#dFJDHD%?hlQ>Mw5)8P&ab%t`fonR{IOS~$dfJsMXBM9k(mpK6!cJ>+?l z-@@eCqrIPgxq9>dqH<$nPGf1Xtes@EbEuVQe#Qtf3J9O#=*kjEM2o2+| zyduf1zSOxC6zg1*A8?D}$2^_uNiGo2FQrP!Ro?R~CA7fBg8JN@aVM$3abu#Oir5I* z1Sv%OuO7VuD&X4V^gMsSF{1Pyyy zKMLPcDJ=HW+Ps-s8|prL$vE;RWV z6=-M$S)o@m#2d2g1h7t;6FpsJ|F~Hi(H)ofBeep);*&wrg zRQm#b{#+knFq@|#IyyR9*}f5VNj`yjh3tU1dS}PG`bo8Y8`jOuo9T^f6tB%Lr}moKa;mbvb%YDd+1US{dy1W0pkoap%wRSB5%mgMbz^B)-?i#}uspzg?( zOm#=xL|!i&YE_Zd9lP_hZBkEV8nYr<;fgwvYgrMMN!bjh#l)ynbVJZK`=hfe%+Iwf z3fdM<6eTKKJ69YYSRHNa>eU|Kl2@tM)drP=t+pQ4)!dC!#qNm9BSC$7GwABu9RGD7 zh&3(=MutB1VSFy2PQtAz}|gMe2HR@;si zY?;pAQ}E=sr$*<{N&V#}v6CujPuM2yF_c>gD=b|FTL~*{0C$(L!kR+-Z=xIDQUB{y z5wK1FJ3Xz||2sN;(Esk|`6l|G4F9V|^leXMZy{2zMP{RhKs}!3Ns08eqvo2s-9pe@ zGjKCz4mm9@t`%p!_LJT%hrkI2Xbpg~w7c){H;zJU@LRoE3!-a8u35FvKWgBbc^ZG0 zQ)f<=mZ&wo(fToKnacEaw}>?>PpuzcQ2nk$bU~Z(@rM%HO3_YpY1awv$~3;>soR@F z?6h2)MueTn@Mmcz_lrC&(jbjT=*>9~72>&qE@X#IpHi#j>^$sdm+$8KpAbLd!&Qo4 zoBlUEt=|7QIX*po(Esk^x!e5@GYwt+8b~coomCwydCw2hoTL!p+po$QnNYHWxM@kp z_!Y!XL&nHP`BT>JM4vvj!_$&HTJ+F=rejbkM`fiH5OLv~G7*^&c6tM^uD>(@r4>*$fHQ962tTo+aHl`-adNs${Rw>XZpe4v81J+$r@zU)w>da-c2 zYO>-QL?xe++GF-)-qUlq!mMg*H}>)DJ1A~buRA7sc*fLn7wr}4|DtKqJerk zUMerD611{J^~KeNsCcYuRSFTC!T7G6IG9 zi0}B%uijr3X-=FSbPi=x(BIClP#ERZ@uyMIhyQZ)9e@0G-!aNJG;9=8L<-({cS@zBqgS^9z3-Zqmo<^?z_OJUpnZ|G~l0!~TCS z&rYqlx8t12nxJxj!<1L^m#^Q-V#x3I;EKYi-=9XsY%vD&P1X>Wje5NHH6;Z_i_EM` z4fUv1X=NTtf|F1$Qq9`biZ7=L!bXX3zP5H6@{&ePLOyKpU zK^}t=16I_rwZZO&aC+arHj6^vE=XH_-8XbIehTdAtI_ogF z_1w=?b59ll?N!cC(D@~bD8eyKrQfqGPOdzVdSVc!BuVEZ+({%jn6X(Y>y1-YEL&B; z_8sR}W~m~aaN{!x2>(dwO|X?j1(t=t*|(E4j?)jkL^?^BxTPbNykYOgE+Xy2pqf?MbSt5p*ksLf2qLS_$-|&i-go3yG zS|*FWZW%6HG3lbvE?n`OvlQ(`KJi6acXVA({&L-OoHrWdvV|z*PEN#{4oi`TmC&28 zWh+?ct+voa3HV~!Unvz=e$hihm+&2DX9xXSR>V(r`ri)ygX90p8UfThF6apO=Ch5l#_XSOG3^f zj3gQ!XBRJh$2p@2Gc#%t7yIb(V__sMRt@d(V<pjz3AG~k(Snkk zd&tPgj06SenvCR=G@o-B;t9^9_bGmkrmEpw?{vKbd$sYxW8Px5ep!W1@Bnci)V=BA~7l$)6Z}`DObPLzqo-IfzqSS z4<$-`k5|rTB_dR!tv!*;DmfY_EB;bILqRt+2E?cNr>~1@@?W=^b9=9cePS|0=0_r zl87)vvN78ip(Gh8n8*}6T2;dp<6wwz{Z&pTuQm<{Xa(4J*kAIr!ugCYWC!RodQNDN zM{waBeMT=n;&~Pm^x650JoX^*f6x328eA)l;~ITNux+bauTp0n6G(8twObOR&q93y zO^(6P4!I>1UM4#!)j+PZ^Vm~rXuM1&eZc%MUML}4mAMurslOMw#C{lkul4jpsmU;m z4h~PBUh^TOa+8(QQ4RF}vXKQr} z_DFvmr{n${GjY*>o(2oJSi~^y<1F&$;f@(tXtQ5dnk$+fE~i(htGc~{VJ(+JjcrV& z?Sjsjc@(N~3lumQQM0K?5u2ba4QaWimGU7XI3NuXe8u&D&RZr+^0*1kxu7&O%!dd8 zV>5Eb*L6B61IdK4z@#{{dcu$s1#%9U1XQ{$XtFP4Mi82jpiOgClep2uN?c(@y#2(Z zV+1L_yfcTbB8%keP|+oF@|oZU9WTum5bUE3b(f+SiZChM;`F)rDEB&&=SXov2EM7)0^+_F7B;W?N)8o_FvH5RekF8=e(Zx!>!{_ zG%AgP!gYsEsh_8>IO{5Exx|iDTO`D#kVuX7yRI?7xOt|j+MfxWxgbMe+!IOQ@RH1p zSiz53dmP%4fh(C;M9c8cmdic1zTZ2ErG}`2VPh!i(5IBd44UY=9 zqbSyuQi^YIO)bh-a~izKumM6drx4nw@?Se@slPrE^V%QU2`P&g!sMd9wGlzzh)sy~NT#2(bxi-j^ z=r0v0EU7fH%@fHUjU0Uha^g})uIT91M)(+Cx=5rKhGc_pa3EU?Y9<|H~gSHAYvAbh3~*%BSgZ}4b}C}Us=0Z#X80Z9%T ze~5omGIXf4ePE;2)UpG1ckEE3@CnSCX6F>w5ck#}?GoU)=U5hx{%sekQsdCr^vi|_V8;@&%VihHOsTUXh>)EG7b;kZnY;QQ7i$qK` z6CAEeKm`o3!JCi|e^kt{!OVL}s`$D`9UGje+z{`)p@#C-*sfjp3N;PgJ9&$(-}^_C z*r3}!?OJ?(7ILlnE&sXX6Hds4df6*Jm@KwWwvac5VZLs^?W75)*d+#)@5uOPo9_zS zUP^=6cxq2tEHvJqK))2DqCLKW--P0k4@4624ovSB?6s-T`2z(Ns_K|bLC1Te_r}g# z6`NS5WN6!+X|lp*z|acv^GGqmZy4)*q59)iiHr%nh{eYdYLsF9xkWK3%6LbBv8X|DP_4@c0rD8yfJSZfebwr&+LR znogrF5&g`!nQ++zY?z^4-O&@8FFD`ev*MLZ7LpbHff4c&vUP&!9DLB)LMat^XP)9_ zh4LN@UF*SqL>bY03Db{2qV5h7&tY||kkq0kR05Hyu+WY)aDDwLMubJ|#&An2#)? zgn#DJyBNN0Kq3ACF-{N&hRk$1QFX11ww4{?V*zfeXzKPko~>f+M- zhZl7iI4>eEUgVnJeMD1o+j%}gSHeJ{4EHtb1jU@NcKL#_2Ze>5-&ZALT*RfY?{pws z9Egvb!IlqH`w(x5oq@@{egeAr;;h9?J6(`q90Lx1(-aODGzo&<@rW>xq1JN*IA-dmubRAs zSy-MDX1#(s`1eSP0vt!b+ikwsib=sdpPCH8WgBwS`GQ?0;^CAUS{LU19RGvH_;9nn zME44wJ+5Qc)x%}_QL@aAazXu02|f1{K`H4(9V9-%(64!dhxH*3{Cl5TKt;7L5o20X zK_kdR-0=gv%$j+ISV&2GXq{e43(YNee>?_p?ur&*{kfAMw%Xw5&F?9*11@eKZ(ohR zc19Wt3kF{w`!NjlA;$~n@N20LFrUN4jb^w2X8Mm9p~a^+wnB+;X^PX z28A$k_;`4DXz6kPcy_ycwh$zYB*!%T>gb<~2T56qdT+YQ__6NE{%8Oni(iV7JvQJJ z5>0O$-_2f_K1o15Im)8XE~4iQ4Wl1PmqL2ju~fB{?2w)P(JGLIZ*hXFO*B1?Z;SNr z!}kj5^xnV@`Si|J2Z_hoEpB8ulQAI}5sPH)s9^J?@2FT6QMDrtR#h)kfda-p?7QzD zA)8PVa#arIu`Ht@**+c36tSsPn7V`sWMfjo;OaHnLGZB+GwC4<{K2nN9^Qr| zOgLpzi4@@!QZ_t1Ji>6o!4jGc;=a%GnRPE3zp~u+rHv}*{t9%RD3DMyc;^<`xW=|4 zwg28kc@{;e^S5BW#xjTBjs-;>qd#L!6C>6Svw&a+|WqB>&#g=z$K(8_Xmvrh#cGLy8+KSsK`5jRSVxQkwOZeM{V)SF5 zOD19URWr&nIp=kJEd|O~-b(pf#mfpmUFi0Ibx09Vqv64AsoqIgW3{QZGz?nY&1D3C zD~dueg2usAC7gtC_7!Ab*vQ~E6de0>*gzI?EB7J8^&jU#ds)K z3cql#_=Qs;YxWd?kVkQaT2uSrL;tlbWq&qGI3{!?2y9W}se*@m{h>weks3$8hE5zA z`nQHqL|KiMF^~J>xk(oYQ8t6wCl|RrnOxI116f+yza58X6HV6jq$S;74xo;u{szRqS{@q|yMT*{U1b)>JRfl zNsCE)eijG-v)VUE1&UBP=@jLLlhsMQnx-2ZM=1uYi3Gu>6Vkxed zE2aVR=-cx}icBK%Jc)H^^gKaWsY=PMXDE$K5>?NTn@RfjlX_v|Ym0@1Q^3)U<2SPi z;$%aHPsY)a7Qy9|2@KQbF{h~LU>W-iZ{k!ki@1`5G$YjGb9Rs#SIlL?O5Wbnxf(XM zYnGDr=6W!?hYqhOm0xOz*{p8seLZY2)V%H3A<&lr6tXb&?E{W&@$^M!^fH zPGu{hLFrug(sh@?g0@NUZ`N0%i5ghU6(IJQ69v9*>XlSXVc!rHV5XMKVpVnCmDqt+NyX`CLqQ;wr_86xGA z22;#}+x|xjGgKI}I^8KXZpyF;wWNr}D^tfVrlE~CMpJ({!x1o4N!;xX1@^`-Xp;p< z;m+Hck;;nCdKX_lqG~((d^tVL%K$uCI(q##3llTxav+qAU}|1 z#$Pu5&rE?wHDp{Qn}E}J)tyj>fE5$z9!bgwGBr7m*ojkaemQDed0G)9`(e9Nb=Vk_ zDk2|%^p4(Jm;x0Ajr6cbp%x6}viVUaI__h?6+A{iQ3u{KshP9+f2UtsRXl|&v5o=k zG>7#BL97(u&+@hM*bktVWBmxUxN8>r#f0-=O$DLpV0S<21^0=%2?e3zdHyurSal=a zl(iZS%8=L7Z&8jj+tUBQd-)~nP?d}1GfonG4w-Q;>fc{=+9KpUtW1P935)Eu7B3kB zJTSFjk^kPjRGbXY>TGZoMZSg545w4niXD+GD~Sc%%}aSPo=#wXc`Rl0idODW+eIPM zq_@LL+!GB{%PkvdLm&R7vnqB5j)U67OkpFkGH{YDLugYdH>0I?;Ao5Usz08NgTL5% z^lF*PdnR@$g!XN1ug((Wepq+kzdh$Y_mlkNg&W-it8ZgOQ#fi_J71&$1BZjwGDn=4 zPLmLpi^}1gW{_Xi@kIUg%6m>dpNgt)QO5!ttdXGbwRti5j7QyQ&BT>w=su{L^7$A% zuywG#^D!u}tZ_AkdcH@eTd1I?Dk@6!TGlfs)R|n_I$#=gUiTtQ--u_6X}o0pU@uoG z)2{IGY(zh;t%<^qgc&NM$c*x^X+F>TVHf1-9{rM7o(1I@ zx3_4l5d4U;tz+86{mRQMWK@Lr(eAs>i%(CaEccR4w7zL6-r<>^Q$+7AnM0;>)V9Dr zGw&&0;`cRGH`zZEq-1Wd;e#C0D3(`gTnSWEwv^mjH2kA?a8+=h!<0)UXNB(2(%EP` zeg1Vf!`R@VXBM<1WiKy_jx)=2h>}2k$-3>@s6Db(@tRT93%B>k&ZooqZP3BZmm4Rn zImzZ|cd%EzW!`-_U8cQIyY@d4 zm!~Xio8P4&%+$+AB<=Ar*YMig-~%!u?1$sKWp=>Hx9etc(g@t&c5(swpBo<^sx!i^ zD6f~f1u4Gz$=?v-Zk#-T%E2NfL={GM130^Oz#pIg5-!~Me2ajdRLEE5Hwulc>{r^Z z?t&&j|k9!Gjdw~$J;#f903Vw@D zpfak&#jn31-kZ04ink~(st$9+_+nSY#KzBW?dn-x{Lb9iZzMZ4P7B%&z zyL=dAi=6#@?68@|3!|;b>O&WA-GN~NS#-h~=g~MDMXG7PS4vllF&cu%6MHkq^lxCy zPOujze+tfj(p&3ELQ)gkO5~DP|AsmKSSQe&ZdwH{i=Wr1jzcWf;dRjWKtwX>3X=uZ z8RRe_x-6Hh!7!e15|fG~M^jin)qSyJ7m|oCuVFqyv&g1Lja8_V08E^irl1B}Ej)kY z4w+Vj8CuWP(JUr(dwV+Bb98vTb@{|ICa}KKlD9*t=?$6&yD_e9G@7>SP6aA`e8p^J zcg-rL>(5zCj3o17i!3hLQ}~=ezge{3C9?^Z*O3NxZ2kwv&^;n%GYB$T0ynK4IS@-| z8l2tD^2tb8=GEN)59NoR5nE_I1;^K)i9N;zw3>$7xLW6JPXr6#Nzu;F(1qn=)ya@B zw}?@kZd$|9rsYfU%}265#r@Qz5_?tt}s09@#owumKhnoKpF0JFV2`+DX%t!ycJLqLeh_So#O z3H-n6zs11r_?OlntCnxr>PEPuIIZG1j_k%ModuO)%C&QX-YMsw3RO- zRHZ{Zg~rdQ05LW;__UP^-CFi(6l;>rx^}HW=GMN$F*rccePIbFHFPsV8;(p(94X2jPuhx6cymu5Mem%BRpx z4luf)Hy4}>#7pV~XSX_4qk|CX(`w3XRgKs9_Y)bJgy;VK|H@%#vVNBMfNJGem@Rr$ zB+|UY%5-Ucw^>)wei31K)z~@+9xmS{(DQkrnr+iJtzmU@aQ@w9Zqws4SLWLwS{NFp z(zt{Uk)QPQTBa1AEoFn?&vog2Rul{}tvGh)G+nz-cNe|Pz3*VKGlcxV~1Q4Ew-O_vxkQvormIo|NXsxKlMRq z*yDBg{j@g2*t3Y+U)(F1`+YGasKIe?k3y8Q^vk+>VyDLU?dkq=s8P~8kqR2z26Xz* zp)lGMm@tU#;c~rLF0z@*u!;FV^e&PoG|qFAFIvyVZ&hqlZ^n166&%IC67qknveAPv zMn6)St-2N%Pk%PPPY%R#p}g<|!yhfUnvJoKfj-`?$EhtzbI*&Vgr{kZfk0J(+nzhG_J}DJU!~OuoWZ)>OaZ zI!12*-9OL9zd5H|dbZQzvwW*azSr-mtI#^`Xsp0jKk+O! zCQ0mQF4!=uU)D`%{h0DI^{0D}Blhng)nr-1cS7~-wk=Z^*Kog_o@tThPqvzjG!3CW zm#Q%nSEDdvO~XJfRG5Po-nfjNu3!>eX`r&yEQCY5f8Na9=6^4A^JQiO&b_f_L?D(wD|Aa=kiT_12e;iqEw>W^2ao_A`#ZnNyp`&i zAx%WjYNM{d;hM16zL3DR7*#F!^s6k{y`{Tj4vK%L^Aq1Egrkj&6vK(_LNjepYK_`l6TE1ou=U zx0^4C0LlVvj;%iR6|%XtbqbarB`!v)%xD%?wn&z%_?mrvgr7PQ=D_WSM+jvsX~lV` ziuEbmXct~1OYI#H>KB~taFE?d26BtjHyQ7nq=MK8@;OiV)WT0-I^ys-{>W4(G(Oa2 z>i4)W0D_@V2sx<(?cR58o4f!AkGF@9!hp`$6Z;pzNw(ilHbl*At{Sd&hx~$4E(s#R z4!txMrXphq`z7S~2bxH{@YtldM~d2=sDIuU5fo+bE$2i5=`q5^c*)5sLW3a=<6@>F zt7q_C9q$#1#}zegG>}D0;A=u{)rPq7HHDHD5@)B_&iZO&n!{-)8oBLez_W>~L z&$AtduMMK{=R^Hx6=RV$NXb4j*0F1eo{BSjFpU+}TdO0DH+gOH-PdWSd)TTI`c0Ze zfT$Ab2TL!Ur^lv0)^syiaH6r6c~Y|XnO9}Vy-_wgeWejyH$GK5oFcXU& zduO}1|8+F_Uu}ad&6b14iENkyaWql&WjIJq51F!?@bT^>%(cSsX(>F?mY#M)N|}-m zDoSRC0$mW{`E~EC;AsSz(WKPHtqGzC6lhP|$DgEufhLdqu52jEI2773Y0rURua}>C z5x0}uEO(N;upe))h*6U2NUnr9;rGOx`{!Nny}+ktPn!J4rQp4ySLUOvkO>}B1SIg< zv`Mb-Xx7L#)(%(fm+|y>_VL`J=jYBe!aMGC;cwx&i4(@M0}Ic0MHI9c!Vs_*&j-G* z(u19OB6w*)CgU~249*4MerW9>*YNehaZ5m>_UF6a+Y@&ELPQ{Z26%hASr2vNpU5qP zC4bfslk+PVG=d7{f19tJ;_vND>0)H0GA_QF&^UUy{e z1}-`U%|UP}w+fq+jS~wwq!x%Jfms9{p3EzyHl;0{?iR8>E*mdvhxJvT_?BdNH$Yj| zI{OArGb)QA$5_#P<(N4+9|bI;1~HLK^APSSko5IMPswOIWK|fi{RC-mxO!P+(f5LJ zbESn|C1o4}tI25U=)k6XvWuLue0Ry`rTYiz@wo}!B|C(m>yK%Hlj`W!rm*ZF)V0$S zI4O)#@>V@Ms4US?B6##B*GRt?8E?k)6G@$hg8k;#tVdpc^i7eJECSR)^-P1>Fs;+Y;!yZ4E1* zQ zNSlEYiJ5r{^n(9$5F50V#%xdY%ejTs>6jp&k;hZp92*6hFn4-sE7>7kzf{4VliS_A zdlqXX6Kf%cQ++IfAcn-e@zN}TU?@qE(6h9{sa$(&L1foSk`k>mAwAVtdswv3q?D3l z&6t#c^aFRQFu|f43YMA~a`^6rgz6K{W>B`FhMw`v3JU9w=CPt@erSiaA`!O7D-My z+i_qh4@mr~VS}j*f!LJSrtzZa`u=^ba&hvr;HVpQ+BeX_sC_XjAIZd*KNHiwGZslI zO(^H6;;Je{I&^`!N1Q;D&&tD22=l=r?DyEp^P%*zzTS_ZQk>Fo{x;ECWYNYSb}1Cm zUqWnE`$r^41}A?M(CdNrwQgfVx;;Xr7*GlWnZMvh5}lu9zoKx%k*WC4Ka~mwUp$U{ zG)P32c!DsZ6)Qw8>Cifd^^34lgx>kfNu!}r@$#MT*`~onZKqec$-rZbO{^Eo?YK{@ zofqpHY48jUz!gRzO$o_00~+ZJGT>`qvVyq~sdp%*#b^FC`e@QFQ7TnZF-AbKlHoS~ z`C0TH9~dE5U#QNm$BYmPwPJW=dT)K;@2}ziC+v${kv41QRK(3+pztIZ!V?U0dd}i; zpESpUSvWU)X4C``1@JQ2o7dMo&s8&L0k+Xn@kHag%p3p)snIeUl&cNM2A zvS-$ir~-e;!yC(A96CH%^+gKS~_#Kwz50kKj5@IE3TFthl}4Y|q&%q@!Yk7Hip=dU17FZJpsXI$bg! zr&TB*2$?-I60IJ3d_FG4Mu(b^BoK5SsZUc6aMESgKo*<1n*}Q)I^YwI#Kb{r;iF%g z$Py<<1uGZHT1g4VtxR;{FD+EO5lHbNVh)y%4ZY5$$sU0#Up0uXF}R_~$0HH9Kf&v= zO4A@!gMhA2LYWI~KYKTC3*0BLuH;Vbl@FEJPp^<073*9}>N%ZM2o4pPMMfNeYUnf$ zzWxNJ&l#+tA9N=J6MVpy9w-dSi%X^PA-4eorHVvaaaNcGxhFcUV2ru{h?!WN1#}Xq z+D;n$w6wxcnLGDX${1+Qe)!04!=(a82yQtsnS$sDu_3}kZ8w`x-xNZ2O&l42b}@3O z&o4|oP=eM$Xii4CE_3J-Pe1G=W}$8vIpV>RAr^3ml3T9PrHk|))fT>LuF??#hEA>P zE{!9DF8Nv04k2V@4nh_Kl4f^3{~cYEOOV4?9V2=uVrrtyTpt=ks!L@?nO%l25~6*s zt-UDld=%9t0oe-tsB_}O`g%W9+8RLjaxMz8^!^2gOml6J+Hl%2s52WsfwfW{tuV7T zF5w}b<}da$D89DnjIr0KZVqGdnSpw(j#Q8eLkVh_x?*ey=K7%X#-jx}AI?(8lz`Tt z!H81RznFlQ}AdnqRXDx9H&r+V5TOAro~R?a+mZ z2sXC0hK<^4s=chL43`VheA9@Kp1!1-t=vhnft%vndu?&<3DVZetc&BxK)&;G?G zP@MaslZiLXQQ)boEI5Q4r`t8fC%n1~%d!Sgh_)oCB(AKCGKJ(630GV<%W9H3a#Udv z>N_>EktI?Y@p!MTmMk5Dg?(1xVFd7PAZe2%7gVQTxf6le?2tjNj$?GcX+)|=;xie4 z{gK%HCypvR4?DjGIZISx%kD3%Q?98l19M61%R-zwhqeEF^a+g&axWLobrgW|tFD_A zdtJ%kyaB&)K;S-@Fy~&13!5C5x>cS$UNAG0Ju-GT>vjU}=jXWLj=sNi5qf4o2LC5+ zD}&J5VSEONEycJRQ(EdCtBbAU;FJWOHpW{4R*C0?t521o<%z*d9@U;&x5Vcx>MiZx z9ObJ8wgk>JtK5rn5f`E{+8acnpT>h%IuZ|E1&_KbFsmg&3>_mySmF3zVF{7F?x=P& zHH^63MKbt3tR#p0DU;RWERHvl_6H^F94S|4s!!>2D&(O#J`^L&P?^$tE$VOEyeB;d zJf4qMEQSx{6b{K1$8RS_Gx(eoC-8p(ZQAEnzkmdx#SQr+5(>|~_Jqkd-wHiv2e~T= z-pt+=Wc_lSbjT0FWL&C9fBP!L`%;O9{0d6AP=F>pOY~y#U|~{Pvt{58d@iq|JhI9d zMM1)BM^AMDHx4V`jG`#US)+|8oMWqYQ!6`wM7cjIX6^e+t{>SXz1m*}0&3kespScB>$ZPTtYg&CbDUHw** zq2lLxwGO#WO1*-Q7{pug$S^a5MVkD%#7maqQ*b~wxa>z&@F^#qhM&2YX~m$O9X~=< z@rR-Dzz62Tra(;xcSRG*5^KdVQk+=s)9`zO2S8m-^Xg`xNuX_0!0h+JV3nY-PO&U@ zZ_x+#+o8NaK>WVDd7LI1xb_j^*bt?!VJV!lMjQ%k!C=#=ZRM(uoEvM}d}q`uXE;sY z(kll99sh9o-;(=+e~6-m0Z)sj3FD!WmB#)#bpTyl>HRke7eaMZBx)q896S!`kpzHW(OSX{`#5wc{3o8Vkc?X;)Vim z`{%@ytZbsZo*UqSPmGxmp_M7rHdQTmS^s|)PYUevTCWWxUz?F5=x;xgf}mR4usRdu zvi+OOp94Sro$sjZ&F3z8MI%OT^gni!Qt~-TeFI_K@kI`5f}H;Ccx?pa(>+5aDBWbbN<|oGNcs@CE+aZazh)J$ZhTO?L)q}Dq@LARd^rv z93*Xt28vrTG8NloVXV8NU`Ld};maYv@S$}es}2haCJghnX_-1qSyD7g!mu{TY5qk@)e96!C5bwMYPV;%aX-?vIQgzdv=~`?lfVdX zbb-VLyX`UL1?%Y2)CKeM$;Jqd$-+OZzdMd1+XD@ia%-o0@pSY&TO9aLLI`c8Brzgz zBj1nd77`xG46O3MC4?flqGVj4u;-1E=T`gZipiK;c!#)1AD{6b)fKLw+T{kgusWt= z=#Ice(bD71;*Ox;At6T~_aW=c$K9ludfyvca>fbIE$p8-DzoaWLzWBtjkJTZu22r? z?77i}S6$RDke!!$(dirsej=H%b`Yocq3UO-&T!-SSp))0N80o`Xn9bJmRsf3Cs>J; zS25U|M1+iv4UJi-yywG?T8K#sj-&Jy8X@h`0T$oPAJKTmpSyB<9NWL$h1g8 zOnIjEhpHzZZ>cA#n;F-hz*x7$b1DtAzEX6~1pGg*uMGS4gj~fPcDXuh8);q~br)D# z2z92Cz=d$v!EBs||G*lR=?NE!63=MilCd^)n^?F>E%hT>tq=7SXn2W#vct4JVQPNa zjug2*me=kz5j_v;#XU&Uwjs)(v2 z>DH?h!Zy7eNl$@vNq5-A#G?KCR>o_HR26qr_Y>AgYF#v3Cd)_KW})CV-rw4BE&(0E zLqmmU))yR(JysR=0J860(<6ZCS6gT8K3VQ2Vo9`K%=HWkMp5F;%h8kb}5c`}wNS&$WjpfwJqF zPDv7IHMUkgC<%o`xB%l2>wEZNGEaA~|mK<^&rZPOlw zRstu$SQEy7v8%S@?b76t$vkLRO$YC8lXr$^2N&nL2~-l% z=(cc;ndptays>v0A8liXdx>UaJo4cf&e5U3p&PofzA+)VnM9k57YKF5o3lA z|0-*ufH66PoGH_0g>jBz>C&~S1dmTo<3x%rDwqyx9o$>TQj~8&BAL@`|L2O+56#K@KB>#JE`;ZC35#x572^y zqbBnhkoC`=NxQHyD0M_Sfasd0Pv5W!hPvoJAT3uKXmDkmhMDX;)6jzL>0 z=^3m0@5oyd(tSH%%C`+bhxPh*99Cnf?si7Hc+<-`%SB_Z&c2zMSr90N=CZCX-h0LM zo!DC%);g`6ILETpwOSxVype66t`0?Q%>x*u6n= z(|kLS7RWKIBu-qHZ5sI_7?rBdn#|+`1fdFJ%OrM%k#~l{j4O5%k1=l4-N%;Q(sy}w z>$~*Hk|y70WY{MSjxYMhbiZ0NNny7{q6RgSrBb_wqW?>+_*pXU^^(jzt$juB3QwmTdh~ z3$LOpb#Nak@m`IdeZKr1yKiX&x?`t%jpt;=*r-@Gp-YOX5&{rbx!0gK{(&w zY(rN5ts9}%)0CQ zaIYqMCOh@+EQt|}b_rF$sL$K!LX4eaz2~QaYrCG7j;@f7odR39hk_`k>Q2YW_HoH z?Axm6Y%!a4wz8R|Wz!LklL`N2ZBW%tfAynk(p3HX|ef2~#&9Mx^;Tcj}Id zB@pTd^e4iuzssH+jA1t~cldV%p9rcnKny=sm**X#<`I=l-T48Aq*x)4aV}PC1GD5e zy}7*}+m8T#zh^DLov(mU>^evNKX(*{y65rvZ&(5c|D(aKVgoh35x-LE9Ow$|tZD$etJNf%CN_a7zrv`rFNZ zJrLTNdpYs;PoH;%ss&9?jXyR{grBe)Z@s^{5%tnHlknj^PtN)Eb|9B~2~GErXxe=+a;;IpJyW`rud4yB}BmZ3Neth7=|x{u@nUde<<(4qezM zzTA#^315-QlA5_xA@)5|@XLTH+<|fYkVbkf1mD+6eBfM3_HJt<@xZGvv9)#z^N5aj zU(iKkTxdvpJk<0<&Br@wsx?y^NvC(`u0s_O2d9(vL~p+RjzYAdV|gNYN8`e`kU=2m zVE}>~rEDTQa4xPy<-6|gejr-YKhy3L!}#}7VvfIDqOBCfJM~PGFHXY1$xN=m3lz3X zmk9@(-k9a5YRa_Y171$UuhG;s_sKmvNZ3)19>?`n-|=p!QI16I!qz2Bn(F3P5CgA< zSMRrEFLU|IG3Xywv&FhSN{Ag*?Nvn4&E6xaV)cXMx6_Me_4rjjzrQ|EB~61-?L+uI z<5J4AmQxYl0QT=Y*4zOpM*!?gM4b1XMvECkzH`5qJ5Q{ySPXE?23N^iHx^@NAcKTu z8=sARaz8aj+{?ncKH0^+r=2b!(QpCtOV#*PblRU2?|hN>6n$9N=L^V@i+;=2nqBY4?-AQio$xHjB6y zc?Ozi3wCZ=B;5EV7LnLfw%=fgz<#bgoLHo6HxZ*JGXvA+=-o`W8JJ@4u&6HcmyDU~ z-NnLmO@&TZ0b1SM^o&&_m%z}0cePH>=VLKd+AEb`5CJrORdR;>YuwylXM*3)cP9(( z-RrLi)(&Wbb)}d=td!&%oVoz@k%}udO8HxhEb#Jor#T(Wbz!_$z18ij!(Io0w>hrA5ve-f&(B4<;x%X4Q~ndedI3g zLmdngUXpA`Q`V%vt2*z{Cu%MWWAv1AT%!E`(MH~k`&d)96Qvo-Wa#Mbn53W>%vleM zvRQ0K&=T1OqO$?xqF|CHiwj~l!gH7H>t4lUii4TNmGS4@YvY}04+*Gq82r=wS@Q?o z+~bCQLD-if`V1K)jH-w`e`che=LsM}fN{9#+V3ndL82@MCrK*p=XX^)ppVHc&vj^S z-uZOias5<+tX(;}ghE=-l?ICNCP5zVWgK)=AXfOrl1@V8W6YX2`hGB6 z0EQv&yr0XgBaukfS=Z40@2}3J`VqAhLr82#{N?8|6^OLx{bB0?XX%Zg@5}R6W~?B= z*X^-OCo$i<_x(n1@7qSd`6#2&VBvC_hRwsqnxbGXL9geNsYRA)J(-<l>A3i>9QN2y_iQViC_RCw?&ap?g?{A~y=7DM19a$b&5B zC2NYf6R}p`@4x2Ga(B8~tvG1#vUT`Jr{uju=1en24%ZqjU^mVh;o+t9I?YI9rtoM7^xFerP%GPgRpPn|OJybqAMru8|oS^Y$3_nN=iS@bainq$w zFVku;v}XEILO73#vO_fSIEI#8LuF209D!7iHE4ugzgi})E~F1<32~Gaz#r=9h?%I` zbC!q>S(-eih;}~4CW{+A+Hiw4!F!;gQ{g_)UK4V`+g*$=0UL}(4PKZBg__u`!ClB) zv)AsQ`_rc8b$5fyJuzMPq9YY-X0UuAmRG`7c0dpc0zh(g-~HmLYtX(r*}h-SelAEE zcfB7Uu}`_zM=rd8+CO1N7%a2LnB6?xv9~+Rq|asogYUd=g1)!42j~D%@NRn_#vYGR z8`yn$u`Zt3TV5y+SE1%J%RcgrH*c669}n9qJRU1Q&&*Y6{3c4*#3SY;TxeRhlEA;g z=x%WTiOr-4eKL~Z?udWuHk0hKyxj~%nSb?KK*rwheEATFeXH*IRfqRX)nCGkozw5F z6_F|-F7R2U^tdhRy)@xMWn2(+5wJ>jkyF%w_ZYO6x(^ULUZLDy$FWFuejxbSER`67 zo#Es={M2T+i~m!b(b>ZF!>na3Fq2-)*s}aeYZkG1@6a90NGw^7WwdoY<+R_9s!dHLN; zDKe<7$iuGAr^Jd_k6ns`KuXG2`A|aEfWJc30Ef27kbZ?ABdJ4&iEp zwXPGhX#jqFGaZ@>jdd2KR|__t=Ke-uJfhWPaRypg!q{9$Kounb*sus4*3{{Ri00r(;~)9` z&i8H|HGqG|?>!B0W+0$@pU_b@sxZ#dka+xdR7Lbn@{*_MGf{gSbusYb&Cdda?F8F2 zLa8L#s9^}|=U-I%TDU|0_H@CL8%yD1gH&$1QelOc-Id%5+p0rhLehvnF700NSryoE z!1DHl@n#6$W-UKRZvD5^xvBj-fjU9(I-c@UwV&Cn5w8(q4&g(cGoW~hh`BX{A=T68 z9Y>Z-WP5xqn0X^uzD{PO0{Hy??m%9*;15`FPPK@RJjSC5^F4KmmLO zOOI)$0!6tD8#O?8_pjep8;#D5>Dq}eMQbFN32^hqqsO{d%ZwREvBY9UfxACnM7#9$ z5_$nNxdD5@#S4cd6S8iV+$q_B6UEFoLC6`1u-FrTfPn5{um(D9eFMHd3l93Ug*1%1 zHY|?*JnXQjqQ>=+mT4@ifRUBV;G0CJLW2Hfd_r-PObr&8wC>IV-E)i7FhgC@wF0no zl2&y69H0}^SpSSKCn9=JwIF|4F5nCE=iY7`@bmBA8vW(Di&Py|ZOb9agx&Ay2~r*IEfwFRfa&su zF~ORJ7xGDd2}$Fxm)1u{V`QSXa_xVjGp<%N#xd*2B%;*++xP$-z$TToT3- zc8%jtQ$st7mWo2c;_%`T5t^sl;TCf~LhK33Nbzz;k>#AzuJyM1z`T!LxgjMomZ9So7|fSNM?b4l6Cx`_b_;H}6c;{m zWQ!W*>U0GjLGD|9Bv_6oH9?4!)f;q?*1;+oV2E$B#^n8t58kTwr6155-Uq_57S!)PxoPk1y_A5-4!7=Hu%Q@u>(UmS31`%x%Lt#`@QAgE9pw;K7 z#Y*VnW$Cn_G2pbN-DMc-3&;{h+S2Tw zaD3N34+Nb6MjobVbTsx)$ZWp4Bn7e#nU!hax#MSG;gp^xS)2&&e3LAYX=%1K?+m_3 zVnLrIiO_2aXyi|#&5BGHJ|l;uNI=Lj@FXc6L}z&SgJIaMQ#8Cy-8E^X8^vWewn9jX z{GXl1Ud1w76CipeJh{aQ(1hg4p@kDMfnf<;D=J8t z5Mf}zUACmksv?v~xC;uY){^7k7TWE#l36)mx&d#sfaf0a&Q8H&blmG}ubqTe_9Nxx z9Q6A;J;b0hv)ed*RfYeFy?1)9EN&k!lkOxPt2?%B+qRu_k`vpuZQDGtZM$RJwljI> zV&*#+b2T+n^_zPCgk81wIeUMe^{hpHAek(!x`-=F1a0F62>u!o zcpvrwMu_?T@8Z@d_T*>xd(4&$L5N5#ty*bxlFK#1X|NYi3w7afm1xx?o`G;~5Hm`% zVFBG#0%Q#XMX3Co8AnUm?)Ywb0crl@*-s&GY)SKp_oFRs4;{uz(ahV@{C#^4=;L zVGBc`AQd2Q$z<^n4~= zXae%FNPxtt4S=@RCS*&U$;*Nm19Z0cADUi_tpqUr2%7xkr@F!Zpb7jj9LCG1c>XJy zhLb5lx!yR97Tm(6IuI$62)>)W0&7O>^~lV7Ge@zP_O<%X6apavY`{yCc!znjd>FOu zJ|*ZhoFt^;B5AiASS`;!9$Xd{F?vk9FJ3BEtJ@3)8WP8nQJxtJM|qo?!2}FKz~}`v z3(Z9v`x4i3iXUi!D%3{Au$CC@Bd$5-$d8y2Lh zhu~^>Dk9=<>(K)7fElDwiA}*!pH_`kNLy+O6gUEdsxB*zR)T^pUa8DR-}-6G$ymjD z1>Xj0$Cpr4Tbg~tf&k3EI{0|c5OOu+oE*-JRVZk1=I?>Oocd^GLa2wr#MHk)x%8$) z{uuzHETITV9wN|I^Y3eYgXDmb!Gs`kgmz{-bxcwykx#SqPkee9h29$T1^^p53*aTqYIjI~c!V<0F;a37SwPz+A=w&}jFXj183H zi{nLcpwkE9n-Tcd(G5T$>DLxY+&l_amdB*p zh7JZU23epMES<5=aD-1p7xI!^Sb7S*CEE)0C2%K8cL0zuI}gJHlOjtloo9X)JJJfn zjCb*mx~fj{kBT`RU3%>I{SXH!RaHzJ&U75@)agU3dF3WccyK$s-!B`3Hd6lfAxw~*iMQ*5Q`^%y-A zF6sr>WJSGyaaT)Mp8%?MUfyWwjy21gHWf_>ep59hrrayosE)44V=yo$Q#)(wJAtL_ z#r5?ruTE@Xq||NNf`^Q7IORD*0mLNkM3=(MzhOc1bxHL6xEN}!Wg_AN8Y3djSBeL7 z!w@;|2=xWa<&NeIP#LDoI8y%OnJJp-XSx+F?ojIYcXs7%4qB4OD)qJNEuE1*cEeFD zoxjyqbl@2-ZK4hYds9P`Zq=q!H_@{a)ubm&hDixD96M_E_HBwpKljClNt zL7L(ef1g?WN$i)FBv`mwO)5ho$C8y6pOUAET45gzoUq1hFHDc0MfW=aOcO`^LnY^cb&@L-#F{9eP07Hyj7vG@9u#LsA;~bUmm!MlUj7eM zGZVjK7>vdEdM)+S?V?S;31##Lo@=vF$Z9J;p09{v{sVP{Mw)-p5G!`pzFogy znj}64NFS+*NEbJWOV#(O%Dd2}Wp8_K)~CDvpuLs1_ z{LkC&i?#*yrt^{8N&WlQO7N&>pq*l>xNF-;|?(uQSNVQBUqPG0$y4hiY zr6S!gx?e2v*3Bu9xgJz&5^im*bFhKb&HDD%ku80ym&ArdqK~ll)R~8qVU!o%e;IJ; zG9YPjPLNM|)0tRN{LK?hIw&cgHMgBG$hce&`l~p(>G*t^jM4!NR}ScfzebXImpQVE zUz6AoMqS*(2Jjz{&;T9}xhc#dD|pr!z>f*LQ!LN0h`)o zafkzhcB6-@S@cCgEk^^Dh$Nx?7YmW6HL|(n>dBp!u(5FwF&f1@di8{?c8p__@I1Y4 zxjGP-uklF9y+)(kDBj#Z{rF+C=6=9-*=wYJJ7&mshCf&LRR5UyL&SDOYG6nb-gIm{ zNiYNkP=m1&ED~;6!c=WR&Ym8|Puj?1f1)1jK=!3vLp?gh46)RPMJE+;lJv zNe@&&9d7bZRA5wwX!*f=hGP37LJG+xYybH~^54SaliK~7&WDOHp`dyjMy#th=o=m~ zS$WGr3!%fj^7%d%JBqpa7IOM26EUu3iHQqSr5~bEnp2Oy{r+1mv@67jWbzxbxm7WY zr>WaZ3Lo`j`+LZe z1quWfdduRvXQ$yJQ&V~gKedLSVQ7GBz>j`zZ@>7x3+R@Y?Hja(}FTuhh9JqKo(st%+K3 z4X55Vcsy*YPK1&x&^k!iSkaF(qV7Q(=JAkYwFn z6NT6^b+Sej92f0xa|jStAH)s-qMg}+4*iH8Xe2yGuU2G$;cauiV30G>KC1`uv*2~- zA}Q@ZwlE1^FS%>({vFfVB^+T9-oR8fqcEH2Ccw4D6xCfs?VLNM-}aImLU%CIR`e)! z0w_ip;OR6vp6cc0UZ)?dqA9tdK!K&X#y}q2kdi%0U2MIKK~AWzhvuMz+vNM;6CIEq zsa5+p(i5qo(&gW%XOuSc7pw!&vFjoJ{l{Q(JF2D_XPI0qPXj(3ml)cVJj5G$cblHf zAPzJ1SBJ^Bn+Ivbseuxnsy#ptY;q8~&wU_&^3-tz_>0XSzJ}#iM@V=865SJc4$b&* z&qG3+_MOHxVHwVJkA#WpsR8EXRP=sUOQ~yfr|ago;1<3qPVnC%oAfbc=+{5=e%oav z4Ij^>->tb+z|X*(LG;J$v$-wUHPs>|cyeIMHF48Uf$p6|?9mrMaN-pWoB~e3Kw4Z=VFn7p!amGFGSsnf;tm(Sac)?7R6EY&G%s6k6I_gz9=mYy= zQS76cGeqG0Qgq8SL8}V>q5b&?I*^!2t_l~K4;`!xOsH8?mS*1S1T}Y7Tm&hd@E59^ z)&fkfW~ZJ|>vJi3+XH%!e=f64?J`X(S_jw2cq>HVPRHCt8TXn#AhPOo=9{Y$q&kxzkz2II~}DQM#ssYYXcG* zfB$2b;T9RtV~2;jAe}TH$>EVZ3poAk@F0%o7jYO5`n%6Oib$?nO1kw|I^Kk#iVi!K zxa2i|R>JjSp`eBT?kU#WJnz>-U3$ypsU0{fqJj-5(cLE2%6v2$JpUIRlw<!Q;nI(5fGj`OG&SGL0{mw3{`STuO~kaYG$jF7 z!GmeqnYnRWfZcdM03SQ-4MB+!3KdWk$Plbg@Goh1+_7p|Crw+La@f%TWoYt{IduFE zbTS06?Uvh3qp+qTNuvBcl<$&vm<8l>G*J=-w!Mw_j{Av?NKQzxI@Hibaw4-`Q8SHd z%R@EtL{iz;u&5_fgzxXGpPHcc_nC`#b3^#QXY*CO`pCc^$NWx4t=!jFsXT_(-Tm@% zcAc5{dc8l17=Ew*{{n-#%@tqaKAOOtUAusxw*qrPNr24_=B8Y7PK}g*=(*)f^cqC7 z{N6xkA^wbM7(4qwV;F3 zGh=jmA#S%T3W21VeoL_KSP}n;&KKTVQrQ85w&RqhJOQyDN%xZtH7j-s^kDVePs_C_ z?yS7zk1tVf?;p)inH5+pS_>5-*p0F*tbX&0{8-X`vNm*+?9POw2w@=%@Zh4jbB*t5 zWs}E3T6cf>R+(c20mDECwD4Hm4jZe*u<{SLYDvu#Z4heEyC5Psab4C4WcMG56hKRo zL!$tMGBmA2*o7J%ndhl}HE3P5Ua>^R@51Cw-Ny-p@RVw{-JzX3EBT#(| zCgs$G$69(kkumpmD1p;ND{Z$fQ&%3gQ@^-&`YU`4&4?FxcO$%!R?j+xs?LE278S_M zm>xSg4!=Yk34x|@q+7#Qsyn7_ty+Qnf*0ZM){EcONuE$aY=q|gT!aEwAw)CiVj~xQ z7A3pc#j2NBe;)cZp+WHz#e@PY{zB>ILVdd|VXQk(mn9+sh^?o&f?B-heu;WO~k{V<~Y_Ib~}XSMWgty%Er_KmEOkw zQZzveHtipkQZ;{a*!ReVEPzk#h5HM73Ffyv&T(*O&=WCd7S%Lk@dkt)H5_S06sLd3 zm;!V#`84el4&~2amjolcyVo#R9?%tp=GEPgsC*IGy)Cx>M$UAQ5$5nAaAm06_w`ul zBA(DmPe{~vTbe&5L)6Yh4>kTWcsOk6sKKrOMGQIAJ+x9m##LvmQ~ihqRt{Fr==A_N z+P^RPdR`m;#@Zju_qa+s72bQYbzv!e;7zep4AygGx1Lb_s9$}BF+-_#RJ87u@75&5 z%@4vRnYJ8?1GR<=^h#|tRl6*DAXITGnv}Abdvoki3r~kdEObnN*=;-Mdo(Ak1xqzT zJ{u{5sm&ZNZqqRRTmBs|N`imlxhP{nb<%%Yok8HndN7$h?=Rr98#lhuZ!Wd7jT=R`2?*d@1nXZ%i8z<>rG*1 zZsirDA?Yeofq^iaX-8J&TLYy!17<0%9jT1^p*_L6pBrw|D5m=dDN-Q+(Ra+mF4F|% zO?kx?)L}BljdcEX)u_@3ObdPm4;Dg(+e*A7#K`aPBY>T}qw_BODm+>sGZ8gZJ{$UJ zULdNO-MZMjYhls*ZtY@vh9}Cjkx#v4DMJ(hBvg}%481jgXWz$4!aogzOO8N}dK?FY z`(B4l`VyYSU_~ISGmnP(44Goc#Gm(#J^o-bXHZKj!EgP;dn#IYr&Rz+3^$sv6}^?& z_4T~5=3>M9>3ZLvF?|tyeS9@)^YwXX1Cv3}9f%a8X{ZUDv@zlP@N|DaTu8{Wiyw?{ z06pY|`YtX>e;1c-*5u={`oL450$1tvnMnmiA3YYiF*lux&8Y2Ps2??#H05B#bR0Ai zR3g=gpT(^H+0ISx%lX8=UE|2?d&q$Zt=)T?-cpo}bUBPbbhU{o8?7Y6EsIY6ATRqp zg&K!*1P*YgCt1Qrn@Bj6Hu!rs{v=l7<9)YTyi#iGX#RlD#&&@snqY>TU^1OeW4Q&@ zR`snGXPbfdJ-b@BWou_%qYG$jcVp`^gV_Uah-Bi?47#@$6f7xDRL}KO7%WU@DV>{Q zPGh0`24Av5*e=*0WiW)RqeSkQ_x~-s`YdKNf;Ikp+S*7j&_=rls(BJZ<${%WPF?P>aSVdcTis||O4&!;b{UtKg z!u;xD_I~y6`|?;my6GOD@IJbqZd$Xxmb&`sVK7YZB6PDh#hz`>klq;a3_h~n@%WyX zZog2WpR>aqd#CtS6B3{mgRdMb$VgOFS14{)cV9s)r|#Y{J4U{Hm8OPBv^Iwc7(b zxC$p`FT47;G#gLQ1lmN8*D&KOWGU`rFk4ighPDRJB26`r8;#jC`HX=xXcyV#d8tjP4R?wsPVsHf5mod~9xkC-Nfz~}`ri>bj`Xs({D}(gv3~?^ zK9PBXsZ__pGRq4|TpUMZi;-UypjsTRzyhxLvM^-^C?|<|n)d zX2%N?d?2%vtli9vF8Tq|K3U5_&-qp0&Oo>F_l`{UZ}O(k&* zJjeaj3rA?8kP-#By7v1fF=4{Qlr>_*pc)p7iB+!(7Je4#2>gK6`${My4$mx$WJq=B zRf1?2qX4%J$4$a{cyU+~g7R*Y*y06JIh0X3+OMKBQkZN3=39yrE@QC+9-d&biMA%^ zBkbxf|6SILHbeVhH9_j9QQ4#~zHF0?iW80H`hSlnjUC8_^ZTLjk<=32^%xAhE;rH@ zO0n|bu4-{as^2Ei-@3J$MI$J?cWS6imLRM4!R7Ha_@iY*a?^wJ=HBg{1UZq zZ38S>HDF*JwN3l*=EQAJmcT#S)V5Prpe?^mKkuPho_NlHNmG^fw0+e2?K9QB;^TRq|i=&xKXF7aJ)_ zB520A4X9DqSE`w&z`wkYKlC2nLw+aACPYxvt*kmwbA3(cu-^Gx;=%ssfb{LzGVZsg zO69kUkL+_`5e3o_)&-}8bFG+U@*HKW0V1)L zmt6hw8X-n|Pkb8}II)5O!;t9Lm2Rk0(H*QibgX!9E$Na6KhmTC+%Wo+b3l z_zgx_LcGV0h~mReH)I!D_E*7qzs~|%LPXbd%xi+4kr&epM-u@o*iB|;g z7PaR}_fNSk?9R^}8TKd}hT+{iAKR9%`&_%vUvmF%5irRAC1Czb!2B;wz)1g>fcY;0 z^M55^Q2tjXU<&?!O2A0`mw@>%0rOu1=D!5Ye+iiXCj<=4?ictIdw9tqR1bsjMP=+& z?Tg0a=h63i&HLBqUGI+J`?n|_e<5_j`3MPlHoW*HC>&Gj_7iOREcQ88dSC^t&#Fq< zAm+kGFYe=dn;skN=Pld{Le6D`k2P2Op{F>0b@oFAvBXXNcF1-!b~y-%3m4VLuV|}T z2j7+Lb)2Fw&YpD~UD)Nya@Jvg8=arJW@B^#vI_k0$O2vhN$MUi>xAH5JkzqcbSz;-be|p@sXjk0z zxp{{e8ASIS9(S*=uU9^Imh`UhJU}08^&(fsKf2N^&NjTSt-o52 z8uOVLu+?stDU(*TV025*;}oNqPs*VLrbrkZDeKkY!>-uso!MU9wVtZIT(veAf12TM zF5);M`o4KTAg12xIw7Ur40jlK9$UrnS2kxFVbFf}l@{{W`?u{4=L<5Yixp9to0Mw& zD?YCOBEoZuDkD`2Ag0{qW*W_*C8Z!T0%DBvYYQZ%G*sI!nRj4EL-hXA);9|6!a(i) z-Fj+Lrl3>LpNOpVrw#%5*j~T$f8x`rEJTa%F8MS~CA{QaI9N1m00I}t8c@JthcTbe zDFvo(lax>fdE}aLk@dc`V!(%pB7m#4lk{!w`bL(5x^ND^=(4@7?&?*!DIy64_N)q; zvJ9o&Ry*Bosf>ja$x~Vjn_1EKH6!eTYNvaAS+wmLEJ@cqHFo0MUktz!__d7rQbj;$00!Ho>I|Od z{+f-PGJ)!rRiADs%QF-t`geJ8dV1mS*AfXeEv^imf;PgO7lkptTNu+$>SWEz>$VGw zY7xVm6RCG|R=K~^4=!JNBM!N27HlBZP+|5oxC%{-^n+^WCKC4WFyY6G!+@UBF4P^;}sqp)RIIt=4ZbA*z3$h@4 z4ke;!hhwKu)l&WY=LKq(G4;}wJEt(S4(yWtM|S3o|!S^YeD9cB|}|6u)(cF_0$SmcR1H6_(|=Y;;ehXq{? zqHW|%WP(WicPwlP#-5Ebwc^}VzjpN&Y7gwB=Ky*T{`TLK<9*SZ{jx%^&sHnv&n-jk(=LoAV_bY4uJearJP_xPdZwUzJvxZ z9k1jydraK%c&ebC`{E(a*Ekd4t|7B-{J@SH(8*Bj+h_6VZW^lHXyk_0(W+1kc}lYa z?#KJSi-Z$hP&|OitX?gf5;uDZMr5ajr8*mtjL`dD1w9^%;9XFU5c)kkpnV4MX?Hy| zBV?vU{bgpaJ7O%?*PLW;=cEp{cKMDjct3U=p~<;mi%667OXoHovAX%BkU| ze3*&#PSsc=Q(=$DU_$}9fD@XNV%%T&s|{6~G&N2cq9SeV07MOxL?M0Jz=;&>iFSph zW1D6L1=c%$xc80wS$7?8j+7d_@lFF=VWnfHd!`F!}C6r(OwFESVjp^cP3(!}jS;8Xd6hIRX_6slE%ctT1LnZN zi|?Tj<2CopR&wbDWf5}u6_8o4wAGYx7F#O`NpR!iVEdql2taa3lEJZzCW;f5;gXyN zpe+qt#T`Q7SjMCt|07~=PyJ9Ip|8(UsE7Mgzka6i9B-5942PoG~G=<#f>m? zn9){U3#D_kvymw>jxtw>+}piCi=(!y4B|oOW5o_u;S-R`an(OV>Z+gpqlT%=g9pAC z)P|KkftAQYyWKBNqDtM-KzMrT3mJ=;*)`WpIT(VRx2^M#w`qUBt%lw3#08#)VtK#$C;8B^wcoN^T52kuQ*q8-npnh za3xv$?Wl`SRPYyKV|W|ZJ4=(qK9v_HO@a3y)9bO%`R*I_Q|FQKr_GJ28tAeMSaAK)s2#jr@!!&KazOSlTspCBN`dgt3?~DRy zG{4)#REx6Dx;*RG$gHO!hUlR+$A00RSyGr}@kOF+84%#W6_vo6!sauERMTHdG}8CVEi$%+=GMP^795k0p$Yzh4Et_n%FDI~ zmn25&DDkY^Ys(c5J+daJ&f33@xz_iVQ>D6}v-)1Rc|m9Y&3R^&19mIYF`tzcbpG^x z!Q-1_r^PtJ<9qE)3NDid)ysLleTtFjz(iCv*70!yG%&n{&*USv?>z%~te&i)Z?Bf8 zi`t6{?Vj>4;+9rwHM`r-HLUQTs&>3;Rv|6Uwc!|=*SI`RA1}*xwJkN39fs-QD+32? zSnfun3#uL34jYkMNPAo*@K}CcveC)0BzDbL(zyTx#CxOhU*{U@xzOVE%TPbXSc-_@ zGF#OCX=YL*h~abbU+eT!m*r~zYM11IK%?RSUHauK0#}Toc6;za!_-QraT3)z}lej=krg=}wgf$jNl-91yR38qeugy)BH4Qnc zq9>5*07kQ1@99?FX%ShG%aQcJ#jMQbSvKu)mBm6dzet;UCO)oOoe%Za?2%B93Ifwss-EFY|pn_(ep68q5JY4nEd_p_qPt$r5aUJ`W0sFYd|SbVpYs8qA>2*Cb% zFK64So`BKUd9;kIjl_d3$IUv8+|N$ek9MZVXMb)@b2DoD#2xD zc2=8eEp*tE2?jU;Q=qzmiyxvoo)AYNaQ4LVDO}(M1TsbM`g;c<7}X$bYtJ+T$K_-^ zplPHlv9R|_*%Tkl#39o;d|sDGjEAnnmZmN7v(F{eSviLs+(x~zboE!M{TlG7u~wct z4YePM9!RlPr1hdU6?Dp)7B}D>_kf$HTdIeNbk&64pw=G)f#mI-J`t(fe+a|6zf-Lko#{=)O9*qkqMt zfu;qVMeAMw1`I$th=EJbO@mUOrMj%kV)_9orTu5!ZlJ-4UR}#2gmhG@@L_a9LH9K6 zsHf*k2+>r-apGz7glE_83Ng^!fMXSAHySN=%=&C^5Cl-@Bq#%TIXleqh{gHMa_xZ z^6gdqd6>LLELzY-67_JoOuNLq#&5le;_Yd*1j`F<`$(`8_~GzF;}WWo1Ot-pa^}R` zA1C^Qv|idMJepPshf=3ti#(C;Evu+=_sCKLbL!4J6n;qT?nWDWQPa)eanm^=S(B9R z`r9mYq4#K{hlMM8tc@O00qUk=d#gWd-5oS_)nitEB7^Pe?VT?q;AqrWslCDdQ1RDD z>vFGiuz&sM^M0{6U2^5#c!M*)N$&kujt(M;?-z5l6vL%l$(2V0KdKi$A>2UHbvccD z`O5_{apUd4VcHdkQXg<{mQI8m9;Y2^zCVO7{!OZ{6^Y_0QT$Fn$ERR ztNtQQuy;lZe=1W4i|gbxv-7atePqa}-31y(opsECVBWkuiJKBNFI5q*DW%xExL2T0 zFgUT&Hzx$RBiZnmI=l>b(n)@ZHpNe05It6R*`rF|Ro0rG??sRmD zebhi=6~rM23{ixMG!`V|*@}D=Zs%KpC}@>RkC}i^^u}X>3wOo&j~TVq6V(G~0W1kB zsA{bor5vI{elKV;V?QyuE#VvWdX6T&?IiQfvvTKUeo0gYY;gd7>TM8}*IAB3SQZ$A zCeA~dM2tk={b1xtjkkz}Ih1f9WvF>Lcq{nF*X?HW&r+G4v(Y^^BjXXeV3av#^xKkx z+H?h`qt+-if@S)v=fTCgB||6uB1K?hqXTQJA?RlDl5i>}*`RY{cGiO8Q0-*rPwnZ^ z6ooT$)KOG;nEzQ*xOIjBP_0tin(a##c0V@69M;rn8&+Whi8XC$99%QO2nfG%GfP@q z!W6~fXDKg&HPYA+I#j%FEgcd*J9JDF;(UTf714LRWDjYIbxlqw&w8k~&S#6cx|me) zvm+iYY-th#Zg{Z{V=-ynLU(WhRE<`^!viMuRDa!v3SYRD#po4JQ4LMi369tM6P(xO z?u*by8~cAsV5T4O&iFRQkDNdC-r)PG0lAZO?oRG_JKaFB^KsA62bWu)_iepls?QK; z_l;*=&zsmK)PampSEt+^7sTsRU-N&n{!&eMPk*>yu9ww09Opi7S!)wGP32BW#|_EZ zkX22k!M{~7=ji_lEYbKq>j|@N`o1?e{GSq-qiMW@?zb22(D#zQFHIz`48s+y(0Q%C zdI_;y++w#mYOjmZ)=M)sMCK(PS03YNS1D<27|%{?xvOBI?NyTf4FaQR=O>)c#d4W$ z$T=3lt;JAn2gy92`JTo$woh7J6XEgrQkM4FR|boqse7m9P!=*7AfDdLd9$EnFXO=; z!eB#VZdQt9b;7)gx?@n;TD#DBrFKRmBv;zb^sMv-nvw2-LN@&9n;|SppSA#*d0$o4 z`Y4B*u5azvApoDT_cNBDOMw z0r4vhOYxPmKHBF_ZHOT*Nb&rq^GVWLl1aY z%5J}{3Y8u#JTM;4tGu1bY6g+%*t33dB%?O0M56Ds*=A1A$mJkcl=cJAd9P-DRH_w! zS_M5n#|(nu__e0wRgh7MIToHzH1_N2u5X^*uBU&;RZAV>FCH@-2Z<5+-YEg`Li+-94!hvB~;m6_=c4WA6x;*JbYuoNgY+ z*5i|p9lPP>Yp3ArNS9a7|Eqd$X{!#UM%EkGTaM%A%=^oGd@tt2M1O!mMsc@0ho0i) z6#1{+6uUHQo|-yV7Yi&0r&|TEG<~)ld4bG?tZuiqN`*qhQsEHEuy%XOJIjvt`AO*N z3-gFCv&J!98{js&`2Co?_T7Vn>|Y^^sA_gt2dZcjoINcogdrRYy?*@XgLV(g!zDN0-5q=Qk+6D@zd! ztgk*#f|XQRCNoOD!q5;wFY3oZ)ziG$DsH14H1@DYN+WvF-=X8gbr`z%a)d9D>(1+U zJR`i%imp`c`$3=ioOjBz48JKeVs>0GfJZw)H*Lqpb*8R+yFLrb*VOoa-`a=V_osMJ zvIOtm{>2&FI!pxP7gq!&owR}I+j7Um*JY3z_n6_L&vk!ipGnl6&+Fmw1Ad<%OGAIcaIz7WuIG=jjBym zG^3oL67!+vk;MdstUg~i1@lzlY;Fu)Qm}rRR*`14A)M|ABd-y8$F(yg`w4YWL6(;p ztqDWa zoRdU_1k5)%MTm!Op5=g1%JT%X!7VSs?*aD(#lcGgMaF#4TIZ3zyTAIwj0^%vFrJaO zFGEfbnn5A!@@~!n#cZS6^m*OJ7m)iywa;%SLw4G4LM|bOQry5igr_5j)U6W-*gV z>M7q1YZEZkVq7?U`wZ+<>Pr1#k|XCcif5rqozQke8YP6l%gUhkDf0r$kN*%VC6ADx zR)WCqY6xJOGeEX^@Y%B`9#cjF%p{UD5QR z{8G5hiXUad!Acfm`G*x@NF{L;;4}k(dN$wG8$m&PJVv6B3e0Y96!B(7xpBby`p-h> z-PfL*2?`C>jYE8+_a+gZ>}@J}QgG(T5jaGtyKx$Oj4R(*eJBdbRfTJ{cigbwD^YB? z8bVUB!w{olotgKb0Wp#vGEDUMZ%5t{j-6l-CW}-BPhC4zQsGK|(Z!`8k|O_G3&1mf zQcEIWi-`0-Y@LW{O0prgRt`LZvM}90X|E%b5CJ;3I6g3UG{P4=hErt$DfJh4kbkQ+ zvGSrYc^vkvLgKlkc<_lfTjdxAR?z0x`;w2xR{X{W-d#AHi%X}?=vw+Mxy3Z}i)&R# z?>(dINEHQH;vKi7FXK(s$e|z-OD%77mINbXvD%NU-Dnh80wXZA(0&9oAlrSqe9S0! zj@kL-okm%bIa(4W5WNTK4KAGNHaKpEctM`1uD?p1l%A3_bR7LdZ{-n;e#P9wlL#rh0NEb<2wFZg3i2I=*eH$snatrluT@+i& zlgAxyU54Q25D=(N8j1X56=&lA?#n`*e?Tc#affRlg6qqi)Ubz7H-Q*N%}@q5D`(2d zTB0|qSBg)He{cGg_XFz6)yi{`T5#z*ujDVAE`3g(+&rYyg02#9?;$D5ZsKn@usZUu z_4kjOGW?@d6=n$wt%A-b0rla=3^az(O9Qo; z7^bK;EJVD1>Vvh!gy*s5)R!XU5<`^VF9T7FiM9uvVEE+S<&y*Edmz3lzFt!AMI9)c za2yK1W3g`2Z%U=Nd%w*hHoUWDag6qR8?pSJq3=HOUoe=F4Fe2>Y+zMZI%g`Ql}E71 z*tw452KGdQWU>1L=*bCMQT|4COhgd*6VbIL9i%tV*^V-^7<561e~d)yh#%M(QNCgRVN+56F)(z8E$u(e%%L0dioG6d?ZwIHXtrkA0z{V&0Q)N zA%~Dc^fDLVj)^|*nE9^c%KoB#EV(fUgoymu<)w+Y#k5%nfYx~(7jzno?_YS7v|CtI zBhNJIpC1?mIHK7T&ljjyYx(>3kEF|}$PPfDxJ^u^E%t>(Zu>F|&%hk`6aqOV4A#LE zX(8co-Ux(x2BNWdTBn9dW2K>C{3Bq1WYI_zu@1S>{4{I+#BZWTU_0Ou$r4rWzik?8 zp;g(D@Vxtv(Zj;0yXRAYL=p2RGqWV-ci?*v7<6>Ti06hhP8kVom<${pyRdOysqN88 zMTpVHJ&=;ggRtCPbK~H152=xyf)%2<9d+R|%+s{|c>x)4U==(BRwB}np-|2H3WWm4 zq5H%(1Owgc)K*~Z$%)~?hzx35O_}Slvo<*-Q<^<1CQZiTWh>^5f-0RoQ9i zKQuH+b{QFr3Bp8Zr*x8sB?scVwTeLzQ^9{qBtSZ2$*i&g?jQd3S7)5XwjvH{I!as0 z3@Okw)Fi|_z~%K*UsO`R;=a9W@D>Ctfj9#lI$(Ky;9pb<$2x;!`5QlY1%kuhi25T$ zACpEw|6dv#Se`(;XsTO=m*EKmDF~ryX6mA|BMBcEJiX#AGL81 z?3c5+DyZTaF&WgR>#q8>>;^hT^cgq;mFS#N?k8P62viruq}c}y1kd_dB4*E=u#T}s zjKvgl;+E4BJ@__pk*%XjHcHPfzXM;U1yH7b3?PJLb5-Z2qfPi; zL0}OJiR-1uV4*&MMbXxU947otjCF~_*CMWDA-B^nTTj~V1UTf+ZU3Obf_e4qq>`aN zRISi`dZwW}&>~}AQ#8&4A&f+3&bEYy;pB!o2nupMy0)sc6;#Ilx3a?Z(Uu|n8&!vv zz#%OdenDf=?118T)6|VfX|qgUfQvh4t+vbh^=S!=eTvR3kd zViAbav@s=^i&2kU9qqX{624L0EG@c|;=f;DvjD7%0pbsx?YeA!C6G1ovakmh=rMiL zqS#ZHi?MmgL^v{1Vw2M3AZh%ayImplEIXxBziaeX;QDQyPBgE$FN4Yx0 zZIk#xGnho|;5IVFJh(%Jl<`UUrx7W~Yy;xV@OYVqmEyP&z4J-TK{3#jJbsV5M}gq#;(c>}`vFioH^hQaw8#yh(L;GmO<$h5;es)Jz~w>dt${g5 z+d4XcIB~K4PiyX?FL9Bv8?3c_hhY0s%_I`r0d@PGqGq!A1xMl|@r5L@8KVqfSwyC= zUVNg4Z)wJv77bhHU5i%j9a0Rd7l`pSBe{Y(J=XH0AGAGf?YpYP%M$M8YAz9-egqyM zIrD6&F;kec5dB`hCL6_5eF#*%5fa7GvU9vw+Fv47hNLBTx@5jnU@c1tND06sZ(frE zo9RrtDCE|`GJ_CIUZdw=6V=wIa*3}+BJzOXK$X2a6i#yHt3i)RkpWAIc7$@mo5aX~ z7GN1;(oOo~Npsy6nSj%AtFMNWor2es_Py6=Gy$aG3gEcC`jl20O-qaY*(y_YCb%IImI2@$EXm zH8fwgA3uJutQjbTUMuy4$nsjkwV}_29aT89z)x}Ap-NbiIM>a~w?gzj0n{HEh!$}+ zP2fv5fu>Flqlawd@IDdWg7_V&XRtR;iM`NdhI9NVO4ZwqSl3MqL(<&^Fb5m_!v)yI zejxuZUSW8C$iO@@DO%|N1eIcPz;C{iBT|MMO4V2Rf?KkTPH?qyjJ_@IJQiJF9Lspz zLtk6tzZ9cJEM0I3kbtJ?dZnec{~G{}Kytq%?@xfe4v7!RM4U|^+~!Ip_SpXR71<9D z4#Pn(Am5Pv@aewz=eDzJ&W8l@>RG=^k`zA+eh7Zcp$brPx3kiJ;Mf*D2bc}c-BFsT z1I*o_gP>ZZ|Nq&0^X|B9}&_y zUQ`<24AGG?(o=L%l1(YwIXZOH?v%=oTZnXd<>2#p(zYGFD>A>PECK`hX^uVL1|RXA zt~}p%<}d^>PiC4Pd8s_CP24>nkrc83( zY1p!2otU+Yc|X_M?kArc*eHs`qsfvIKTGUhfyiu~$c&U7fwA?o!!&&jm@$@2uwX+kIS^PCYJ zJu791A$t4H^HiWumC9a6>a-Mlkj9KrMxqw5+o(B00#6j>mirr_yhsw0FZ^Wuii%T4 zILUlu-v-HDVKW;hw5oz(MGpe7Yz>2QC4&+}>#Z$Ein&SDDD zbQ8?E3fL;q(C=HLB4vwPHy%bCaOovQ7r%U2YGgnO9MW_IvZUI-n_k1v)b{3RM4j^= zgyASMhK*M2_rih;njSfvg32q@M~o)&$c67FqjkLp6(VX$<2E9kYPY0Dt;c}ZM?bLy z+EV^M&S_WvIfSObN(wB}b@+s%5 zafo&#Ao5(_33hkDQ;D09_D0D<)Zw4e3^7Vw`zh}A5hLRzXTSSr%y~ih;K}yklW)Ad zj9TEtW;Ox!A@DMQS1V-Q_KaReK@p$@X7vK|D=@BCCN6$XBj47yw)XIfycb?e$Y@Zp zoSNJQ&v}Rwtk@(&CPF9hgQ6u7qxlfM?cpTpy;~66Ux%J2Q)kBxAsme9Gjo`Yz6aC>p~{n0bWJEHU^(D%M_MDUje#nU9R zt*xq~#iFJZvb<{C=dNa(mS0Pulnog}^irc!x8{=KeE?-oRM-x4O{7iE*WgV^erQrkBc&b-14wX_}|HKIjF(*TG^z%7-{?BuCa`Y0&kvhR@ zU~@&S-0dcpc7U&WfK&V{&0yb=D?We-H)2q5!X`x=k%78@nwwa+zHZW&odr{oCUSw- zXrtFh&w#F><5%aU>jjuK==}S}K)XraKrQb@8CU)tmM8)}Cv2u;WA`Bn=T5L2>;?Pq zn?t^0;(X2cj%z&XM^rNUfmY$zV(C1HG9IY<0WZem_&t22xs?q2PAKjyq8g>BOrY4? z?bV8rDC66bV3q-?^|7Iq#irdbFJOnJ`gclFINf)4zkV6RJEdt}t-M!y#t@XmGm_~e zVnn?DB0|T=m@txs#1#>69uKZ5yGkgId^;5IESl{M`lryW=>>jCC}l7IAUa;YIy(ka zf)g?Q)rkxVuqY-8j7^8ioYYgvw_}_flLXJtNl3GZ57C}Hh$M}og|IIVs^gVpGIj=R ze}5U!bOtSCRJw*9BF!`aYbSf@BVTUXnDGBm5IL8RK3rVLWpr`z^uK%m#c4_|6oYXA z+wZI1r$@S?8}i(~a?&KAZ@&%Bbcyf2vn33%G3B*F7Y_OItvu&@>HI>0t3fCWXr4#I z%!_p6+i!z&7{_&@|)UefMb-jpj%ef`j_|`FGzz=IK6Cc{SDL|2Uq! z#JRrwXT&e*r@H@>6i!m%sx3M+qw&0@@w?Joups z|8Azm0Q?yz#*D%xK9=@bK1g@;(SAX)(`K7!!zqD~oYf86$VHeJ7X`-?a>3Omx)4<0 z`khr;!l-F5Cs-FvaY{l8+f)jBunhU}|(>l5tn6RESgfB_@My@6e-ULR$bToN&{SxC4J|9`2^DQ# z>4IDtZ-_qnnnIrzqa+T|vm<5DrvkUclZ+^{o1@bc6BitzD>8R$vUn!tj4NWVohzmF zok0j^OufJMN+2f-M6y}TXa)v8Gt6S;egp;*+7Vz&pUx@HIigIieB367J_}Qv542 z_vM5>g*4nE;qt%Wip(peAz^j1Sja@Rd!4XPb^b4%y|0c@EzE#oOpnV=*{#N@`CR$G z0nG-ofc>VH^)a}vQ8s_go`E1+>4ftHpytbZFxkhU@F2wgsK}y(z}TuC{tzdmZMe!x zx-w{;bKQ&_EgE|@89EF&&Ia}U-!-oGRO)sx&femcA=Og1dzB34Y``oXZ4XpBnW>f?kbZ_hD<*Ck&Cv6Io(Jfh5EVl#%DB$A+R*-c zef0B`R1s4SdkcDyQYs!L^D@Q)(N5BWnxp22m~ruQET%uIhP-~+>#(zs*LMvzxsc54 zWDc9pGrxxRR6C;(9Q!A*9`X=k?r)R2HY~*u?H>Hqygx74M8A0M0J{2p z5lu)#M;DLLX|t5Gsx*qA8lcePER8eE)yNa53Y2LAW(KU-z`QJI&JC)xBmt}sZ^!fG zc7F**xop_hgtAgP)|#y$+TZ(ndwZ(S;PUm?PY$=If8rSZRa>-qop4%hZS5A#&A<#gQp8dZG{1qglmR7O2BMzVo-0&bt=T2hrl5EVA3 zq-W-A0*hE1Ue7PyelKp>p(6Im8 zKiGb<-v1Br^i7;ISb*9Ctvi8gOAf$loViwO4_nP-IQuIf^lsIFNkiiPE!j*^XK-|` zB9nXGhYy3to_TA+{(-4Bjv{McF2ERHvejUNsGJVAJXvH!5_+v)yZ!?@id#Z)#8GIa z039S-%N>*xw?*&qK!WKaxGR^i9FMPL^`2`b+4IVKHmECKknxy=Vu*ULsEN`oUmN%Q zA?iKGq~gMY+A{|W6#+7?u= z+Dem80L$d{Sy-rT*5wqVS)2})l|`Ho$;m%W<%SNYWQYLkun?ZV&vhWX%H6gOL#WG@ zSm`(h)e-2Ig<%Z^$|xGr8k-d@OInMbk5VMgLUgzXf5z!K+^?dZ-)kR~_sqm@F@4Ab zc#h;;svtf%&tV{e_cKwgLwORgru=ZBnxn}K&SuGl4&!Sa-etGenh1CW9H82AcG;5D?XV!C8FTIh}WXc276bQV}? z2FyBy+*L4|;nUS8*@4p{Mzs>S5uHj{D{pTMU`a>)zw%zKHP!D;mfgZV<{eoFsit9? zGG=hxaBgm>cBB2LQ#dv@VKZ!&J53cUxW${!SSrD+-FHljcokr6E0R$tXnsnqIV0L@(+E5j7FHpp*G4}8FZ88@zW=7N|H3-Q^h4#vu7?I zcc3zW=C$KMJN^2jxynpwX3~UQY^`KNvy}NZn}u@Bgipt{jGzmNdtlSHE*Xj3Zbn(hc2cLLt>tNLmT1e%s+NecRlVursNLKHt)}l+&g*6 zA$ond>zC5SH5pAQy;3w;=PWTdj1{LlxHC3O4)kU^f4Bl*S^Dr5RGXhZNg;UbtAY8k zYPX(Se3kR7nrOWB&~|Sup;XNgQ&ws{%iW5#MB4~eG@Oae`>$bG0=b*(ye-waM9w+j z25wE&lCXRwmh=3~nP(FnN20Sbtu{o2p8M6N*=hQI=31^zxX@ltu`|UXBjVM4pfs}A ze{R%chrCoT57l_op3^2R_q~rpNcX4Ru0kB>j5awj{FmL`+Wqg|&XXr=`R_rV8mWdDnvSZ< zJ%$!#gf|CLnHBzfwJ0j*((nCTxCuFSxly>%u5AxbZXHeLaUN^G?M@tf2sKb_;nBX( zwGMaziEkiUimF@ns=d*0+LWF_iIZD$3oTmi-?+Z)kM`Ur{Wr#0J|j4C6oX6Y|C8++ z{oma=SjT^PkY`Q*KR^Ai?Th=Q{C9+8eMJ1(pZlQyU3A|(%k00OJlUzy|Luc=oi+V` zi08A>|25%nA^a*DhHuHf*2h3-gNnbg?_>^!pAZuUsb~zKD1;xSH>!uwjIfceJOS5F zAMEYnk83Q#X&V8(?fy3%uM!M!+5PX6gS!96!S>FE>#RX}QfpW8( zKqdF0dtR%ZJ%d8ExnO4v(MMw(Ls@Nxtnp*ztpXltgB4A2>Quf`Ul&~%=%tJ;5j^f~ zxnU$uUmy3j>|8)xiJV6A|2h&3((V2bCxS3;L(4Sc!!+V6w1!Jq7A&C_Dxo`20;&X? zL2jrSXSf@|^X<+ix+k^Do=BDkU8I$HAr%J!{QOsXa zQNc^_xE(9Mn>k(2r8}3u%Q;+m_HH_HZLy$Q=c+mZX6F7zmQP)A@;|NrXb)t&d`75r43ncxkHF6`flFPf5g$J8&F-_vfnD4 ztI77a73V7g{b$u6>g5@*l3_f039pr0R2szlS(y(97s>pJZXB zY=~R*CKGk@)s6~;W~IhSZD|WdMg`h%_N)WVf^EF14eXq>*>7H)tIXLp0(uj`Q=PQo zWt_!nkwW0<5<2r$z5YT$Jnrz^&-veK4lvEJ;s%!8{~sLg)$jigch=|sgFJG@4jy}u z);J!cwBP~)_9V)?2Li|~m~5C}4|7Dtl(6fV>(F+34XAJcIgOQnM0?(Ej{ z{~heE@Bbg>scKYmUhc~>cy8OabTuZ84Q9TyhFtE@naFm5+q{v8zeJR8ZFm7xH*EZi z(~WyJp)l%)GR2+awq{gWHcO&)k=dx`^-|8_5-d&~95otr7V+&46J52j*Ff<%CE*n>QfrQ~8{Jp%J?7*Tiz({WOuu@YFcdz;y9bB7NS7g{Wy>wtc8gJR6QXg;PRx$b zH*Pf{;m*QEi)PaamTvXWwyYbd-e{~^QB!NwhD$F?I1$vHP2N&aw{3sh>qzIK)-Fyr zikX8|mAe)7D(LDAGAfbdzyy_pGV3e)^l8}mEbs(<`qZoTc}<1wB@#g*FApFmrE9?wv!0y& z;1tLeD%};;{QBVo?0}9kk}lT3H-a&POu>GvQw6&-*_unZ-p;Ht_c|4E16M$iRW237 zsyjA>^;Ke5b|s}t7olt~e8a;_+-8Zli3wY`Yb6t-TfdQ_amgm$ z*}m!7Q%$w#7HuYBypPUZD*~H8w$f|E5~Ou%%gT2dz0FtZmMW8&Iu+dk-}aoZB^h?h zw&WkaE4F8D7M_LM$<5bOZM1bmRf;d$Q%y2i&(&i#e`C`qYT&%C?p=h}={inO#=0net!UqqBm zq4qV?olk#Jo1L}VJ84O2UpXafUpqO}WXy=Y@0l%f4R=67ouVk$1SPqp??aNk# z8#TUtLAWKBt%q9UB@275GSihME$6GhRMEw7HX*AtQ)TNH63E62owse+4!fJp4Y{o> zAdxN9@vu_0(!bwM_H1U|x)F0#&rx-M`FN{1t!T3ejY~R7nv3~y%ygE;`=O@(3ItMx zG;$Ge=O9P5CGe#+n(OB#&n@-;#?W;o3(%$ZpW9Cw_kV|L|KA6BmRdir4L#R}o@+zT z+Z%dzF@HnFZpkg1?V#RFaNH|e^1u=KOW}!67+vf_*=+pXKbhrp1CDtRPS-G8V1USe=_l) znqi=Bx&QNMv^qn$)c>a*|9N-saGn46VV>qS%yGW|nz2^TO;%T-2FtfWh|5%%Utj#~}Wd5xPA_0#s;lK!h}&J|5SmeK$1?Hd2LyT7-N|NSsex7x2W@1qJ8@TaBk z_iS${=D9u)SVjba^He0iAk+<*&$mO<32DeQr(vJF`7C zpJnv_V7H$CcmH5}?f?HEPX~ps-=DUHCoyXPw?zJ3{RD81jkX3g&2NO5Z7EtO=^yH$ zdY7M2_i!Zr(atAxV`&5U`S54O4T?idE^-g^v^0e;+Ezx&rg6ao_(H(qg6Uon`k0`C6nuni3)P0|UZS8iQPz$OoKdK(!Olw-`5tGH*GCzd z0LefbIV0ovy~#26m(2jZ&XPHz8Pt>@p`0+3#2E>^;Q0LFTu?^5K6*yelxFB(&(2X4 zGwubGSPbA_1-=)Ieq{sr*Swfc2J#>Clh3k2+0Y1wS4ED-aYDHFIN;a0_c$2gEAMe2 z(%gIe-(Da63$vIO9Gx6L=U$LA`U?q#7sL_4167XEUw)4y?MD0m?D^60%jZEF-K>wr z`+sNu$xbc)-|o)-I{)*-JbiR(=NTR8JOtiJ#sy9ij>MFpx6KmoHb8(I4hEB0Op8$v z(sWSjFnu0R+7{3z8dC;sK2wRLU#eShbaZ+Wn0NYS9AQRK!AXST4E@tKjYf0y_D36G z2A$Ff9oYyoyacSOQH^NG2LkgeK43-W;~a9?NJ2E|3Rg4OloS#EMAwvEB@{;-jj#l% z$Z3Y=v|xx{XBuLZ7fC`QG-fnKIJ2?N?0|AcXK@51?TR>P4-{2}ZHGddYOgbmd}Cmv zZuMFbFHdE&YXu`@kYXvB4!FRhgbb=JZ0e506bl0qx0VKoN*Zhoc0I2z*9T$bKSj)A zLAd8RQ?V25?EekQ{DTPeBS}(pX0mj|grk#eOg-=N@-n0urwQ>Spoo#2A{<4?Ssx)y z(*hz{o8h&64D^^|3PbkQkeybGJi>y=9zE~koDlT((tf-wQ77L8e5w`&AW!~tAxjO^ z5|v%zEJ6_tA>o0#fxUpS>b;@H3+YapV~ll~w`b2^JU=>re(~z)<@0x~BW~Fy?f{hE zV4-{;-fi?NzcOfc&PYmUgdB z$%RidSofH78pc={-awXAixvPR2~?188A%A{B&dwv!m$(sw&W@}$R`W~KeP}BP>R$i z{_V3LkIvq_L%y#&bf?&@xEy3vZaKhx^oFASU^m$Iyfc%30uX32VaVk;B8t#v^csze zB$*?-z(xr{V#;VSnW7oUedw~9$l?;3yR2l^xRlh4pq!BzFig7U6we465t5-Rk_&{n ziK?xLj0SdF0(+{fCi5*D`5m%_koN-7v04o#c?i139C0EfCrCn(%;jkzOO#38tlB`+ zS-oaZimwPqv1lK$Y$GGrsIuSC46_)DUDjgKU!v=2oN$4z2+8Fl5e^cPIe@59;f@{s zP%tqiETxQWxkxkJScm6J6mys**;1Uf03hcA$_RCA;%tr*noS4`&?yXgOMYn74BS34 zE(neSbfU1xaY`DKumw%X{d&r%9(!qh}K$hv-b9QVwkaOlFJ0p~U2>tn57BA5#_*ITl2jJZ=R=jQCoWRb3nE zgnQ=BvFE|2RLn^HtC>sK@929nmhzE40^)4av0vj%FghrBoJ}rZ@Akv8Jzq?TI=lQ4 zv}cGyoJ}?Zi4#r;kjTo)q`6YH07;C9z_CQMqyUm=ZlRexp-Wo%h@-b^#esJA-9|8Q zer|3d$`C7HnmMU0b4^zGuu0(flwO;ae52#6->Rl*;KrnIsP`09tE?hz1})Cu8?}%n^B?Q{eV8xr}~%^XBv% zTANZX(fjNODW52tHQiK@tG+^|U?reSH=%(ZLL&`~*&3SDMrVZ-OTD8gisjtkMAF6b zL=0(4^qRo1tcNAx`!6|hL=D8+MNS%G3wic(2146jNV5Rw$!C7)ECCydPpS8?;<>3OC)gg|o_C zd1nimp@&s5h?5ZZq@KdJKb{_;Yn)u^gX$gy9^Ywp&^NCo!J?U;7Kw=S#Bun}BEOmi zT%t~%{Y`NZg*bx^2j?pxD2m5p!X#xk@{JS$?^Sy1Yb}*L4k}*in?p6jRT0$lp39?4 zk^7iyYKyL8NrthUvf8`|Dt9~#DneMG%Ykfs(3+LmI4fTU=(#>1M{~PPLECm&$*v8j zyLHXou#A1*F3oRW)+ zr~lpiFHTc(p=`)zOz@P7B*hi|_P;MVsgG|BjP-+kJYn2?3QZua~6ci(N6 z$IWvifv-Hr4Vk@TmOElHI`jNYO&KYtyE$K`mDU+EAAehOW}y0vvu5C0eb%hcO;9n4@bV;Zrj3rED`42@*8FG_2cd z3(z30bpe`>I@Cd4nAKs~8a?43%j7~u+MHRm;bv#Uf^66ZF2cOHC^(*w3r<3sMf^fg zfs+rm#9yBFdRr|k=Ev{41!@(jWMu%T3Q9^N@|UN*I7&!w%lx3%I2QJ& zl$nwnfZ39mk$ZzQ^5J2)2OR>PNVv4!`O!9Z@u;9 zj<}DNfoGEoiG>R(COS(*^>A@b*_D*0I1v}4xgh-B*T_x4gSDa|dr8taa;0dnmI8v1 zziPWSxaO*D{{dInb(}@?dWiN?bwoUV{Oa|a=flU3(K!)7%&?M2fuFAvHbcod)l$D! z8`kLk(dmg12)DFS+z9@l0vQ)ssHCr0%q8Kyf37r>`&(Sn=bK(2KE2gwhmaLXk8D#iB=(t#|s|2k~Mf8IS@-~T_zQ?o|GOvGax znroIM4oSwz5FO<>oDzRG*lsJ7UGX9M=5X2nWQbZeN~<2pz2T|Y zp6z-#&+X5h;7PFUMTCbe2F7rRj+BO8=|?0{DBDMsmF!Ka%_!Xk(Ev!YUHje@nO{>D zf&W~!G2iLO^O1d@;6m9SWd*i*l{_PJ+|5dUTV!~#^C608cm=WQ;H@?* zm=BR&v4b>bj4~4WodgV_K6-7~USO7>8zk@qT~8?|Rs|PxHvr!3ue^r>#pWE@vqn1? z;K>lp)Tx6&f3Ct%57Ei^6&0t9Z~|V`Q9L1BfC|NHq+~{K-+6s|RQBMzFB|ifV{H#; zvwW1|mtG&eD$)^Qh>j6WA~cP;#FD!6lBW(_s(~Vd(Igq+@X9Nv+_`>NEOgAx zt=-PH$H**}1OHLA`65nZF_a%X6&)P(iVz@N@%TBniItVs!AZfceBE`wXLgR|5acYq zp=e5y9En7VaD)WqUeJ@ z@@$1qD8)HP8&_X*pNn}y)Ww)vhYLv+Ioe1m+%AF5AaEzDg6gtFNxvfpci&--N&t2` zzwE*n%5KzrTf@~RTh}5&eFt$l2GzXAxu8~dw7V0oz$Cz?L1#+szJ=xa%wCK@%mX&U zq1>?V=K=b8T3#4$A$311=}N62Tp}?lNT_Z%D|;L`lkyG^=##n{CR`}f2CW8kFZT7d zwz>cu6{1=JF8Tsbf^qi!(X&$t1ZZRxs6#Zy36~@9_oWPqNH&WZ&ETTCBzl;`jH=TT z+L+mt-+lD2qqCEv?_WG0s_kcxspE{dQo<#)u!cMBcO^;IcCd2*Gz%d<#a6Z3 zNx@(}ywJCE*xo*(Fo_XnlR}%QxmRD!&iinjkoP#vB{}eYk#IkZbIEb=A{K;iA>U8v z#FwYX79t{4+sPORSqT`k2{-p3V_IYpniAH|ypJSSejU`H>4Tn8Db^W2ViUFU^)%)r zjK}lR)PXDAf;tYdT*X!;(bwCAmns!d4m&t9SM@;7n7RkuIHl1hs@Yzs=FZ5t97v~z z*y0%!!d6K#FirZ}JY&%3M1YM!{R>=$Vc*rS*uT%KfWab#nJ}nPv%$1+iMcy{MI&NG z9p}P(qsX~n1gB7EQ~M7N(bp=(X0=(lPnwoJvVyLslG95JLEVz3V4Jc4uJ6^2nR%sj zKaLJuYfQ>694i&qg)y+3{YTxD+4I$4_I4ltOi;lKoFwz|_8Sy9vSB??dns?~L+^s3 zgkZQAje%xh2_6q)T5#QXIHi(I8H^X4Fm1^MYOOevl#>%MAk9L;4c%r%hG^zeCv7K? z#PZD6ty^%ml=2C4^fJz3xe5WLb6TK?qHCO0M&nc~GHrtb;Ek!=8*w(#?I&?23*qRO zf(xa+l9Z8zgmcX1;1;0G0`=`TOXg_AFkombSJk5u<&Of}Y#N(16@7I(fp)gHQ}aUW zrP1O3%h*%nR1{txmAXReRZ8#ezKlK3iOp+CzGh^UX#31~e%fXI&695qy{d<+ekD&U zcu`r(Ci)jbDqF_u>?lc2GC|lF!_w0IH6l(s7B4X!@Q`5%P)`<@^gPaw3!Iz_9A2sN zjWbJ@`2nYK5<@zM-pT3gu(yTe|Mz?E+yZlREc8wp853q(hI3sWd>H8jBB~^;uA$dQ zaW-Lub7f0Rv&!D``Y5A_r~)}*X)J{YFLN7eIc&W?DlK3-PF%2PHhW%68`=^l!X%{` zC&DuaaKEn(WB3C#PGYY_w1h@W$DKGWX}``ZWoU!rcBpt=gnpqTt>`vy%5yfSk+yXM zzCD5gWEC1)E*)8Mu{Mq7gkeEh>pN{46K8{C*Lrk-^EgPOzMeEaa!+gX-@pntgH-1y zLOh%jhe9JX!aNR-3Nf{a7!k|}dqb~C_CstW29a<@%gu2E#~~coKe1$pemZ*r+y~N( zpqyYnr9~3ii&se&VH?ERL=%U;*(tgei{?@qG)wZO8l81RcN;M8N`cFP@kj_9PSw;H zuUF90$2^p5fn0ikDjuLSW2opD#>{quy@Oy|F%Ba-BNf;&wEoiZrnQ$fpFj40UL0T{Ej{G}@hR zhLadLA9wNT+zn_q*nOfGVMeYAtAGbRtpr^{vWb+`Ep))pJrs!YM127e+zwbJpgk0{ z;6dF4w~tO~WVVl4YvqP9fDm&^XhjDj%bKA z&1p(%{HPS--|F(%nWEx9KiXsEML9l@_-<*%yTxX!C|O;D$yEofX40TC;8LaZFAibU83w7lWi zRvSG;4OieGQJO`nwW;9$K?0p|A-Vf4jVxHHQozA*v7k4ork2({Ga%HGFYTl4yhaQ!J*+vlI>{>+Wi1A4{&eXUs!vr{5cvTC4MMX6Ycv z3T3oc0*WccgQIE`SR+2aq>|ufV>)bcxU_mFbxgpOpi9CAebaFVuiw`pBGuX~6o8E2 zNJ{}oNHL&-FS|mVNmW-N%FjUGA-Og$mO>aqNO#M_S_(s=wTB}`h@3_DS|5QyO^|9) zzrhhkvXBfWB$KsBAfe2&Ffq8#mZG5GYT!g(LX_?uI z1g!cx(J~;C%r2bNB`;M9)>@PS{h#D9l+rIIk%Q;|^S}PLzLYYA%*dCL!IGkR&@OfM zk(M~Mc26@5Q)1hJakO>6Ro}#`08o>1Efhb}j57I}3!LU0=`1Y3lfu2B(+?vO2JMkA z$~w8aD*@hCi+}R0QXNmL9XOYvx^vAcgi}hnzMob^|Ns22|6TG88478V#Ju$VQ>4;i z5p`dNU2yvOS&v0f&AP$m)~-DS)j=9KyxosNHjI{CuNn{Ft*u{_{Vdn4?j>tqdw`+cZF6U8FHH}&+pkzFm$Gh!Jz2r4ngEn{@=Wqe zOdyp$zIKo2=m8x!kv=*#W+Ey;Iczc#@+&yQ-@^_2Bf8Er(LgrMrJ z*v#uPBtod2*IcS0PE5qtYiUA7rT6dsN zWh(jup1>w*gzbPR7!lbr0$@BN$vh~jjfv>P6mYL-Mz+w8l#4giT8t^u`+cmW0}uz4 zt&%)efW?+OnJPO7kZ?2Gn3ZHp2AQQwt3xxjmRFIYxS$MA$Y7=-j|}>NBHxT$sbApT zY58hcA4uy&%k@#4X!**SY$(w>*{6(#&Sdx7@VYWiTSuTQq#HocD}^zRbQqed5e;Ng%xQE~wXf=&q2u<)x#t z9KUMOhr2`Va{5q6i!xU$-dATA=u`D-^3hJP8|(%9oM$@osdXQ4^(;72sY zaTaJRW?d+q2T{fYRX5Z<@ z;l;Q}kRU9LGi<^VfjL6uT5WF|a`_iYVggB79j@n8{)cSv6goIpF&7LYE@(nnxi5W0 z;M#hKUQtcCl-8-F9?#|R&ue8BIqN9u84aa`QCkE!XTLkhR7z13Vy(ixh3Bp^*N(`j zn5cg%fjuIkS|+}fC20oM++!u13|#EIRGpqG6=AFYf#f5s0|j>twXv8gF}67hiKMn} z#B_)T5t$8~kA6ZYi#t+M+ReJsGM8lBz3Q4jWibU-Kf#>80x5zQo8Lc5U-=-!##&?P;0XGH<;9K*j`EncfQnn#ips(KeVxALC z|CB1I*Fzh}MlwG(=y=VZflO7t`6!o&eJy`{Jg4Cm5g(sTF*}C2`S^W8!z=Z->O-+r zYB=_9Jsu-CDv{!&+DT3=ml>0s+{YRt_p!>uHJ9Buo`6T0IhnPKiE^nX729MC%0)_q`75C9Yhc0dx;{o^OSXig3m#f1GTe5>+oBm;TxjC z8%w)RJ8H|@FjOlMl!b8KBQ^PQJM$!q#j|q33W}s~6ywC`vU3_)`dKZkwJLq|q7pY* zmaarT2K|>h$KOmD;ZvGKL$v29BKi*^l#W~e_z`k3mse0V`*{u8F-h>;GDhHTT;-K^ zcwQ$PXN`(!YW*Zi->y8pbOdx}bEXEOo^s&BYvTh$%4f3+_eNp1@d$X_&H&qZOTGY^#v{O zR6v_4f$Ex#5NF1&qADPSc>x*`btRzv5SgTU(OG%nqa2ef8u&PzmeHHE*2Pi3+g65P zX-K%>Dya;!zmKvAuC%mqZCS-!9W)Ahj^+}ezOFXbM!9vKD4NC!ZAI7??FWW!U4XPd9e!t=d5LN&y%^FBiBvK}EU~CTY3It|C zM;u?!g0Y}=+KE8r1sF}@F;SjIr!;C+@S@)9@faNC6k(Hpo)L8k&{OKYo+KtcbN{OP zeF^5O@IB16iv1!%3nGr%Ae`EMf{&K3+SuYZ=RnA;=Neh$CCJOdx!gn_59CIo~+})J;<{Y zK=K|*1RX6E*s6`@#>~$q2{35^bmmCSb#T;srQMu+3zF_?3GvnIAZmDGY=QED9djOJMm<|Mlq8%|V@dp#hH?0nMY~GEnQ&5_ z!50)q-{VB`du-7xB7`6SCpr<}xCGL=G9paltPl>L;x>s+P7UW^?0W*wxzPDHEr4CT z#=@8b!#L&P;w;2RE~fb`ME>=yX7N!ui|5bJ{&n)KWg?;ZJ5S?npsURNqYq}v69=Dx zcCweVALpMQH733s)^I_l?)J4{101ZdoljpMLz1H!@<3^O51SxCy>|DUr>i%p+S+Qi zW4!8QTfv#mB?W5dPoH|78?bVr>1?^)zaRsPo`9X*C&6~G9qcSXzj5+_aA*$s?db~~ ziA#djlK-XEC;xFFHl}W|<7eTFxOx6QY7p zaIWr}E-x?32etR~w4~T&GSIN}m{OP0+J=mB;Ju+6~+wpzTF9S_WkW13Z6OkmBBxY5zz{X;2*ItF3Pf<5fM1(vPH1Whb+0UP}{7_ zDMqt69fAeRI8KPQf{;tephyx8?dq5=kI5E_QxAn|kl6Wsl?jmRQ2+fE3}*Or^%*W< zv}jR{)VWFpXXWJ$fh$R<4`FI6ZK+OIUvKSyR#u@iy>bs3ogY?~sm8NfmC`<4t4}jj z>$MyYz7v91p*-M3UkRPe|mC=Fps-U6U@cwUSMpTLzYE7RoUbJ{_+JWyzv?E0s5=#|w>} zz~CIX?$%h!O%SaPSdJWhME@zMAT7&OwW;@^wU5-W3i@xHCM5t)J%@z2)`#-54f4?< z2`=0v)lSsFj>qllys?q}NjYjP{l)UO9@zr91_Q$9l zyc^4L;)q+DU5#fh5*$ivy6NdZ8p@E?20GTsn=<m0zK9KphhUuZn5z-vJI za*#Tk$N_rQ)v%zf%P5@%+-a$t(+d98n09k)t2%85xQXk!`hax}{~-qM zG}sc0G^=CdCS`An-#V(%IT1FKKXTpo)dg71J*I)@>i0l3pbU;m#41kL3G*PkuS9(S zR=JBsc0xAD50wZG3oDlK6y$q1y2A2mwJ-;^fkya)6jhcwr#s(TOHfUz4N?l zy$gGfvFiuam#Q&q%vslY9_b6GlfTQD434E#tQ2pzJcg!oet8r&$d{Y$Eym~l&c-cHbCdXXKjzhcfuoT9+++#(4pH4;N{`a z*a7g36|S((5H|^aDW8dpbFgT!_ro-3zOGw=6--jHnpx(-(W(-Ar@TJ%+BQ|C{K1wD zKf$5d5qQ>Qt-Nu;qPA5AHr81eF;ZP2P{c4t9tT^8+^VaGOt=I>)`*4#7ufPB!U$L2 z%d=E8iV+|bqa$fd({Ub^5Me8>p2@e`6^7!0*Yo~wX zk8J!oLtZ`F$Mih{+NfR(@ZR9w_|6CbkY??pzx-=c6pygl#FehHDA`+`p2E|I;B#|1KX?0s42HCT@Wl+!az*E)Qro11$G*W+p7Bs zLf6jRIZ!(3W;qk*XHK41Y9JqhRlShtkz-887dJ< zEzOnP{47|oZ7n;u3X6&bAjPbZ{qhxT<0%;NmGFI1;ALwk4d1Kb%<&DKK0&ZP3 zt3hCdme5^WRZYDam1E}`1S%Yk>){&PVR|79A?F}WbyLWrGPI|%ZPgqV`W<Ah{%ynAJ)j%3Vq&L8X$99&JP*Jak)=MTg|2M@ zi!p>$>Y1Mt(mL8x?zGxS#ce}we#4-}Af&;PBh8iCjh)w37`@emURd{|a@cSuCdhO! zK?qeTfi|^zu#%l3NAQYAct|#8gm)4fWc1e(D?0kK{O1#XJ~n11Lya{?1ya&o~{Bqayoe840nJa5Xq|{ zAwNrBRn=r@tttJA30j%TM9)*G@7q&l41s=bZ#xI)mJj>KO@7BOZ7sXRI1;u!TQY5l zp_wbGKDM{NFiyJF%aURCK#J5n@xbko(#_W_$5cBw*3N(F)Z(7B>k$%YW0q~p;Qm-l z$8N0+vNiQ?E%|vje@B$Vnh_=1B`gekzgPzBo7B{vW!Ih6>E6_A+^U>>VckTJwNI(h zL2O@|bh6f}+94;-YpI#8MG`)0)}5y+Q$bYMk!W(fhI>^if&jknIOh%m)eRuW^*=;S z)%AC~w5rV$AK?FFnwL(t2$STbGMC1bX&^u)dkJ=1;FU@JtcFOmZ)-oC&}wMSkgme% zuPlyrhUE@by1J{3Rf{8c(6Vtmp_5%L4EHF?&0#AeN1L%rAtf zML$Is!Xfs771Nq%Dj2$anmjP6KUHP~s**UeUG}$4vu*^t5HK!PHF$Serv>9CH^?8} z<=uZz`Dx4k=A+@Wy>jO;P|H7tWf$dSTfYN;AT_rIRfAHpKGf;}ss9JtaD@gVL>LL| zCOl2`A|gh400zIU`e{r*KMpBW=MGssdheBd9f}HKmY{mr3&pv9?PV*3i|IC$c1r8E z6C}RlP(8CueWCeL%DPuhSGTrA+U<;Hp7foLxj6xaTs=DYJr8%GWYWrZg`&l@CRLiv zcKt~UjvjJQHv2hIuj;GG){(CA{*QradJy`r?GnUanxuZRWSi)b)y09tCqc+re73MB zYQHTYV%7urc1Bq5O)2DizuwMke~_V1-;Fv7purMb{%AxWzq?<-*aNT0#jQ_GCqpU0J_>s`sN0>G3dH?VY}Sx&T7`%kGOdw$D9fq0YQ`qjaVzY17q@`B z4|~-aASz5$+)lghd$h0`ey+gey|ujx~JbX2;gOy_B)Q2w^sww6!k zNo~>K%{}^jR3HD)OQ|Fa=`HV0|3R{tQc+yr^@!zS)@>+kP-*F9dt41;I={l4+;I7} z9QwFbz`kPXcFK(T)D{i4C8^pLDP@7|V#=#raVg@U8jy_j_NMrXw1MVZ8Nbz8@vPPl zTvW~#tBDRt8L>}{SCC2`xS&h);iT*eo3?=-QyE~X%&WGiqjt;YVQYo04y>SBDWqPK z+65lX-fpC?sb3El!|vv<)V=ifdXty)qJP}}8xW`Jur&-Y_kJJW^BV-1yZc`P?vdne zqf*MTtn9LuX;0fJZ%18V_lLu+^9RM*Mq|%&l)A;r38xh5IrIkv)E+e1o5H5lmxxtm z3An@8wx0M&!y8{OI?DPQK6^ilt^m&__V(F;bHN-$f-~5TL|+9u%$;YVIwv~tl-hWh z#t(6ROaRSn_LF~!3l8Vl)D}-)!wX{P2KVDZ-^nLc zJJQNNC(jD>X8KCHri{46g9DdB>0yKq6*`6s^ikd>o`20Hw0gdu^#uDjJ;^hWz!jvI zMY%)9zLkarh?B-hJG`F}z10pSJKKG1tsnwJCi8NM5uH5?Kfpcn^Vbh)UcXKIR(nGE z^AE})9#8u|G8AK*+V3Yh1S&sI)III#y;nLxb+PuPM+1SP zo7;&XoE!;Pi6(1bSHUNalydhM+De9wxl@0n!_?NnR3f-eLX0{RLvbr}K2XaFt(Fhd zs!?&<3SnF_qYT3tiEg9KX?<#E1*yJ^Kfb4U0PfF7uE+uRoBBi$A55RuPlESfCl0=V zd%{^rimzJZ9?j|T?0X$lj3y#!EoCkWDTgE7nfnv2t4;FtGP5W?F5W*$d%A^8oceqAV$KMJ&tp%M_7jEMx{?4TU z@y2a?)~s=Py?L^i%N z`cHy6%Xr7|NIPV$*N)5z_mpuQ{Z(M5!ar|xl3B){rF76*Uw`=- z(@4aWZ>s#^7mk34@Pts5rfMSoK_T}bt4olL0x`}yoRn0vE}@{QB#T?9l2mpC{$8ed3~8 zk{^q-UvixInqYMd)1BbegbKA6lNFjROJb5*#&vfAGT7mHzHC#1Vj!Eq zyiL-KOZ&9!;ZQuYz4jg>0p_X=6ocG_P5y^Yy@^#(iN+t)CM!g5ckRW`=AcLUx6RSl zI9IohjS2slkJ(OnH}{n<|L@$~+m5K#mKMIyXLHVZ`wV{CWTCbbG&Kc64`{Cg{V&`t zxM0~$4}YJ*FtMs;E&W#L?qv)YTh^jY@wR3%P=|ur%VmTL#Pg3?ckipuomRRWQd6(EGqPVg@La;G*Tti zaHRP1ayT@0+0F5GCQv--qz~zD3f=w~WxqtuE)oqsm5V7jp&fg?mq05wyeR%+E1t|k z>#O`V!egoT6)9jE3e_?Eca~TiQc@TP5v|c~!+URV$CthRyMwte;F&Pa8qhu@w;T4pkv5%mX4}n%31yyzOf&sy z#n;(bZ*w*26jq?@swh>F&g_?bD}l<-(+QwXkIW~6qIb%-_1=vUHUC9*fLs% zYasD&PRkmJ6_*To`{Vy&FuBvAN zVIFU%Xt07BBMEsD(^MTfiEuHI9KgTNaKqF+ggkaF7o7DD4MqcBl1QNw8$y*5hu2JC zi&T(jp+v}?whw5&r}{{^0GDDjV;p|+#LcFF25iEyB5CF_N^r`Q9+UYo0l{IQ?e21-xlI1aXmh6ppYykDeuf?dk!u zIRtpUKKMm{Dtu|y0AcU)Zr1Pq&kpW*fIUI{+BmsOkwSGA;?s7+zdR>V>Aox_aI%b1 zHH;*LY+ZaSRAj2*zoBA|Pj4%K0}yqQQxZb#^Q|#p1)0#yPYG0PUwA0VQ5`6in-za@ zRw*cqfX;$)fx;rk{6yDb$Z`(R)!0!B@NLaUZI&})q!n|hSE<s{Xa)o`PHGt36ZH|nVhSvhF?OCnJP{baCT06!8q*AB-iLMdwx+A#3s@%@xQ zB(IT)@mP|rWV_Y>)|Yh5J7FmOrZ5eZO(lglV{RPx^~-(x=*U!pSq^o7oDPYEQIF%c=ZVI zkC23<87T3Fc%8%}rWD;17LxQ`C}+7rELi8oAblJg+JG2R1z_A#2_XgwLDN%=*y7+; zGIeY}K0~b4sG2=tV5d?_@t)dma>#1y7ANxLM^s3KWue%-mMK~jr6w^n*JY?F)iFXco z!?zv8r(*LHl88Z|$I77&JC9v?U`(SS;CpJ9G6$KHxLE|KB+(&u!U;!Q&=iGCC0|13 zsG%TNg6PKbgspn2hoEp~ynFbHADcRW6KDphdqzRSK~VXj#mZBmp!HGU)=~}&BdNLP z2bl|tgku(b>{cjKpC@jSVy~BAPsC4Q`^oUp+{P0o{Oe10y!U?)7uW%NvZ-BdxG#hj}|qKV!ToNj;dFk!B}vi&8*bR@^Z>znBYn*W11`Q*b$c2@Pcl*7m(iO&Y?C5%h~ zCeg;5eAqAHY)T77%1sP13LOz&01{-MuQ_F`IDaRO+nr5W9X7Pm8Dp^fvALNT6^))= z)dyXX<^QYV*6=sXJ!Q8VrNgW3Q>G4bvx!~Kv%OXxlDXH{oT)l$>ar4$%#kEiAJvj6hvhW|xyc?(*NmE8bW(?;EGuO{f=_WBdfoYQ;X`i5%41-1?7+ zNx8xUB0RBVm&^7Wlbq~c6}INQ?^IdB-lDX(FI=4U+i;%!rHKNI**_d|Qu$khvki7u zRNcD6Z{mmBjd*4vcFV?)m(8Pi8jB1WAUWb@YP9luAOtI3tc634pji z=^=+92IF|(y*BI~lf?07?nUqg8K-%;WjFfm>MplD0NgUe6V^7jUpt(uYqj6GU%oZ% zcL2}Wg81kjBWMo}G~&d4r3{3+zXLM)Gf_`Qon=zTRaf6labYaMimW*qU}csUQgxfxIM z>1O`^8E1+vXl|R;$c=1^8l$?yMcz&f(`@2=0^A!$4pXB3%D?SAidN9%|0kXjLiTI z2Pu%E?e-LC?e*gy#6-{GJv7|}&Pe7%1zKaM>C6vz4zj&u9a<%%Nc3bl*pWk?PAb~r zYQBB7T#!`h5O3d*#`gT$EGaSj3l7cs88@O|?URnIHp!elBa?V9xONk91#)1|tru}D*UUF=Z@`$t&*WGOo z?$3t%oH}Q-@ZxXWkt?~`b7@b(bo0KQk-22*+>7r@hG5-KL4t}B-1(@TFPn?fJK-$B z4^t;UbZIq0{Mk%X7`Y0eEoL!dO7Q)V4LmDO3}&9yWdU=^@-9)MNI`0jpy6^v%XX&y z0I=|IhV=)VzeHziy|&uB(%12-8uV{H5L&ZP5iI3uZc_>B{hL?6yX|$n#H{$ZD$wC^ zT5YuxFIdcmP2os^GL_Qr95&n&~}32HH>EZRu0^Z z4H_V%mHyzrmKt;0YUm&MV>k~*2yo=Dg^cEvM7J08YR_TEHOA**<&j-GLs6FXg(Z50 z|5_~r&&P~A8qY14W96rN|B+`rInY@ncsfWi+n7{j`?DQXHpH-8G-ivncUTHUd(0oFlzl^v!62!SyQ)F-6Pgtq6^{ zKMXOE;RrYEaG^9{R>+tdpOj^2{ALhRRSpEAM8Vg^F2Y@k1ZnvKyC#7$j#acTiG@_% zndku*#k<*ymn@nifO!CXN2&6Q>llm-Y}O#Ckyw&v#Tkr)xGye?DFQ|i$=;!g+RmCU%6Fr7f2@_7dTBZjBGGzYph9bY!gV5-q!rK){46MX z(l--CJKSlQD5!-Jv@n75N3OJCtk0L@>A&sfvy@;wSXoi|;( zA_B&^09PPxo?{S22Z8LB>q5|gXxu(@|?WRlh9n1W<5nEld$ zZr}n_cQKzvPa-eZLY3JJC!0d`fHZLl3l)ie4l=GIFP6s0ul&@4@WZbvj14tlMmrr7 zsdRkjb~0?}K-C!L#`ox$P|gda8Djy@$P*{BvMBSiz&~wBckzE@Ji`t$R{FKlSDNoi zb6K@wg^8Zdp2rl~AhlbokEJVJO&1fJR=9K0d&ipyiS@%AR!CH>%4qNfQce=p=#W z99`2)L=pw2j!&nfoY7Eh?+J^QDeWq@Y+DS%nK|9R!(3n)7{7#l2iH7XK z!-R>Dg?f0d%iwG-d6|k;P~rSMrPvJ0C{(_d!?vjtf0Um+MURtri2T&(=h-^~R^BG+ z%H7LDiPZuQ>(MHpQF$1`+O(9QxI>fcQ-nRiLj&5Swm`7v-Up%=5>;hmk9@##G>06j zL>S;5azOO4Qtor${mD-1o9zm@L`*ITLKvz(mFDi;(IZ!}wI3>qa3 z`DmRi`}h`xA;c}GIgMjASC}B_jHS=SRZ08fwR(}V){7CaT99IvzLvZ_;0_e0)*UUm zYPWT>4^pI@nA(bKET5iP$HvOZyQmwKTCd$PT4i&l)t{w`Qcy5NWVnhU^qD(M|0MI5 zuO@J14?_v3P?(M@Drt|)^37XyZPQi%Q{SHjUgP~(hBw_=G~KY+K6yYwY9W-?L!1ym z7&3am5jDK&0^K2v*RYQBx^^Ctc;H4ZvnzRkkpEF*e;9UZTdUR%84hL1m+BIbOLN5; zQ#>R}%pW2C{hyJa>ct0)9To?yEobevs?TUBf&u|iJ1{|JaID4PKqO`bP|J~M&KY&5J}U`K4(7sfV+>x))H?!Ac!uBrlgBV z0+gsMY5DJKr-Zv(K$?CqVVD;2cc2656+eCytbScyIs$kT++eImA#qPyQtaO$s_4vV zOQh5j&6Nf|cYSn7uc)^#+^8Q)tYKQ~62mubMGKB>aK{zGF8V1?)Fi@{KG1_nRr z6-E691#sQJ=hyf(yAbvg$Ng%Ppp5-pRE=E>vRim^8=M5ehc^FwDqPU5HB)D02VFX= zC`Ykl9F5`+Zr&g=Kl*Q|A-XjS zN&GsM+FJ%qLMk9r0QncxJq8Diu#-f$J7(r6K)1~eQg9J~vsP@`52_ZeK82La3H-v= zTaL_QA-?BcJh0$kyB&%p6<~kX?T%e~$MQ`tqbz5&c`Z9e)cZnZ-H4QbSoKf()8Z<+ zh+|JCK;@vn1;&#^XZ83Ec%V!28O2Ry2E@L5C!3h>l$*|9B9O`ZuQEJ)wt1itM58N8 zYPvqrm-=vNGf=yc)W9E6auVkORGg)(4xrz_x~oWdGHSlHQXh87+>QN&*{XwPC5+*z zTM*C7@{bS=J02Fo7>vB14fKW~h?qo_;0W|T85%uCi++mz%vmA@Fm zM&X|5;YlQewYJN0`BP+H+^SW=e#6jF{XT}pR*8Pt+^jS(mE`9bV@bug0t1B@Dm2uO zJ58LztTu8Bf7|GS#l&rl8U1>@tJnFP@74JKcD?$@1yXPl>c4!?Yv6n2}*@^ zg$^rkrv}Pg{pnIF*c-FD*B#o|D0_XzV!DsKU zu3@7hK?kZ0k;FM+rIU?aXr&f}`tze=48THgo4pJ6qF^B58pN0N)P|?SakZypW7mmn z*99%y)ss+|*B8`X;zwM_J2?Y~C(FTUf3qomtEIa8UR6_8&n4dZl_9gxLgNBoWMMVD zPp56W>riNSl`5B}*$=l=#`ORV3HuUd+@8KRilqTCe$7b>5|H_>1m|~nwDPfk*mc6e zSW-w~&S{o&B_=o7M~ek#-ub^6WMZ<9XU}*w8U3@u zsr#w*x8vW``pSP%>*Ps*!G~gNb9Xw@K_%J`L@Oyt91h3Qf@Tg){MfuRqb9fo z%W|7NFzpj-J+StzG5hW8p8Gr97A)p6t=UMD&Smr5x3l4kNbd^saXpW35&W$+V+=W~ z>Cch@*j&YV|FU1Fj+6CVMeQ$Z169iUc^72XUCL0X4svGRG5EkEl(TRVD zi+y;4oL~90I#Ic0x@J}4iE2TI#>MlPrxjC6HoQJtSZnQ4zj6@7QDG+UDn8yxGp*;ENI1Pa zcOA-zI98q1Cpz=(PvpW49V3$=vz4yBYMA=fB^xJFYaN<76b39s5lQ?uT^}3=WV>cR zdPA{vf9qP~lnT$U$a|JNEX3pTdEUK5h>xWm;5Fv&!H5;+&YK`Wf9{7Ej-{?WSJI@* z%5FvcTU7r;wqdZ~gN?ay21_WYB{zhLR!R6l*sC;KWy#4IxOi0OJ@Vrt(_bE$Y@(%C-`S?~F zCZCJTbL#JY+fMK)uL^AW*Hmcz64<02=^kgN#be1jLPMpuUwU!xaiId=F6XNBUEZ$AuE#m0`mzv(Pd$i#0v-Osq}=do9xu$aWYsIp zraxcpJ-(?c+%ZtJ74Ih6H~~d_UgJAFPyJp-f|+!(xleEtx&Ka_xh_cub(}h%d||a? zoPwe^?=%9+V!jzkO=uH@z6ksS1(PJy0x(>;5=_;zgzKp&!T=Gm4oa09ssO)MIzIxN zX1#4zu5eDBTK|P?!)Y>i{e14R9xsPD2{Qd-0_^5^hPsEE%dN1jPGATGq^=afs;l3T zI%Al&(%x_56T71nxfVa$2WXnkTIlEqPFJTW8JR$89tHHOb!T|gX-_$sWv_H0kVrU0f}6DHb`rZDZ(X?TR6`BJQcH~ z5G_4E?k;bs>en_I@Xq3;Vu((@TRMgu>1N2B92^>}J?Ji>e9)fomq2Y_Gw|-+(%qj- zWgD5T2bpXP+pD@muYZ)P*txm9Xoo+jc(aJr9yd9MV>@m)P5ugjnSfJKxbcbTqLOzS z4LfzdJ{u8+@u(OIRDzm^pQY4|cj?j1ozYmkp2o(pMet@@PC=W*P-O_z6pJph9eqd5 zRieqo>^YjG0UF<-qTUo?fsFuh#lQx}-hG1|OABrFnydZfE{U!h_A<_vle@~>gGY+^7_Q$Jzxu!mKJ@CF=64v{H zWKX=m6!>dLu&)xdLRK7>Zf#~q`R4G`*!XBp@_jOMNbW3;w>*<3E`L)(BcBo7hP0~A zv~42B)1r1tK0tD zxu)yF?e{grA^&*TAryk3L;451hn6?swhi!d^Eddz9W!^M8&Fn~Z_Nj4Syb*E8T{)h zh=E!I0=cJ~D_0SkOx=K8dY;h_yk#Mgu^}0dww`@>e{yu7DT$Bgbjsi0j6@k$q6Aad zcA70ILY|(zmA8SWZ@ZYq3a)M&;F3Y8jnsN;PBP*0AhGfjuZOTl-s*J%?6&K8C~ko| zV@QopAz}kTvsFBgJgHm{H>oC*#sflevb5+r4KMlm0V{{$e0}|!?|gxX++aEXj*ja= z)o)U8n?4cnNN=bw)YqApUM^UB5{-NxkJDmlCVrSP8x39Wr>j*G=EL075Q|qKRpbtJ znIuakP!FBd(wMCRu^hpvk{JpnD#zr;iqcWT5+LS!12%QOK=CD|my<)cI}MTBs{^DD zdj$b?UtiCE`wzlCppNCEZX`60=Ds>C8EAG20=1_qF|4^>qk(cK*imjfHFH z@$v<6WhB4hynvGO4ikK`jn}FtzxuH3X$MP?bt=gooE|oJ>9#E~;iINkE68;yo0*d> z;-!w)wvGYH`&BK8=@hMbk_)FFQ)Lo94C8o=a*bjsGmp~H!`)WxJD_Q*K_c2|UD~G( z7E(2ckt5klFo8J(vra|u`J9X*=f8>~=Y>k0*=At2YP3vg1FE$|Z2BC0&9tW~8jGBGvBG)UVT^-Z&2 zzkGMzd#_#*$!XF@FfG;Ul20_;zH)hLP5_(!>;4}OU)TGMqH-3R4ItDTdm{ykdp;Nx z3dyXcdt}`PJU1psda@hw&*jgeG4i1>>6@d1XoLqlzv-q|dGn z<3%Dwv3`8|a|16Y&nTTz3J`M!5BSjB1f`l2%uG}nY-4c#YM?x{A2lb9guMdCbX*Gt z5Wna-e9aa>0#x;7=)Cs%&*c4*pQZ#(SkQ10K^nTZ0{o}Cw|4&_)CD=e3H8XAZ$h2^ zn^2FzaAZ3;e4JF@guNqvhlG`;4<`}xUhNwD+5=bTnIc~9eEx?}-})xhzpFaqYJ)7B zM*bnxzxH--{XRdQ3HUbw->;Ar+VfKvi7KkfmVJ^jtN-Jpw<_H03Z6#+!^I=rz7rES z6P{g^c)iz6u!2n*xnrfGwk;%-N#9<(SLjRjAawLq>< z5{(?=irNF752**n$!~&HSS{rVHZ}>sy5NU@6XeHY$!p8XW_}R2)<~JExQQEwl zfxjeUPZL{Bn*F>i_?v8Z($k(uU1zmWxQGVXV|y9nbxmbx*5*SWKIoGIFnd2ErIoaQ z!eRT-CBB_&f~-gd&zUd_3#YI&#iWRFrZ6V-7RSEJI!RmPIdoRSzM z+enQ;6akf_;eeang}`$ij!Lyyrm1w5u%%m0I*mbV>y8;253)WAet>Gb9clf6&zLWN zh@bALmKPfBJE5sTymVNFWj>7NseoxkB?)!b1*LIQrmBh;=&7LaGwobV$TTJtWgHpL zv%e<9ByGxdupzBparg7u9}xG9x3=Y*nQYV6dG0H&r=BFP>8&<4>dXsQlG-o)l;9-9 zIXi&$6ZJaMg$_*-_55HporBESi)@baN~0eMMa&_P`p7mX5>rK~xN;O^@Y?MwNJix?iGAI%XEG+rm|2pz)-MZn%e`*u)8BBgKSo(6S@P57e))=tEc` z_?P@DgnM|kF|!$>r&{K7LUWSY5qP3^%Wd$Og;w!;;^R;!G7Pa2KRmVEg@D-54{_@g zMV-d`YiV3!t)2#=F*ctBNc|qQVbY9cj!VGKRIM!{rIp z9BCq;xQlS2ypg36>y_$cJPVh|blMSsp(bjrnCsU$fDF(Lc2N8&?L*yVvRcb%}o?>ZW2t5J#Fr|2Lve^>0L-_P-JJk^e^2Ba!|= z)Fb}`QJ4Bg)Hz{O|AnYG{R>gQ<~MeU78y>}ubf;?5Iu7_Jw<7L4RQ>QPCa6Vda6(9 zm>#E&NrTgKgc#$S(jAQyf)Zyo00WmMfxy@Qy9@HyOGb<~UzJ2bPgBw&u^2ONu74UL zXelSYMTi`1C{6qB=RBc{h*M%K(!Rc33L6Nnw3MiDcv_{o_V;r-y~m7bgm8FDuf|fM zVuOiK38f>L$E^vqv2jrl)>sxyyhb{jxVTp?*5ww=4;W$CibGXnI49lLJ8ZlsJ^tN6 zTTE@igg=^JkSOqf8#Qt6RF3*L)A`jRDq?v zFfsNOK4p;RrjF_v?eR&VKiy>w&^^bc52n`#@k6h0ss$!huk-y^#^U#uGX=j$g#mP4EU}r-SBYQvWi|Ur>uShh;*%-^&tA7y+$6Y-3neW zX-Rx7TBE2W)s3)*!*Hpf4omlZF-jNQmw8(!T=fwd6`?+SuEJrO_`z!nj7vOWEn%^= zC_Bd9V#kP1!Xr^MF#?lDq;N$ip)T=uaSOq#FKH&!!f*}qC-&?F_RrU{t4+T+9StC8 zpe3ma6zYD6vP#O?0@**T3}Zh?jce`~Gts`bzzn~v6+#<xNz(io09A+;LyGOm7+a4TsaeET$y&~TW2~CwA(<#Xp zeq8W~FA}jI%8oAyA`)1vJfwD@7+J2iWOS-?OHd@+Xu@K_5He30m63QH*U_318sVfp zX7a`-tw;0nkkT7Za<|f$Lzq2jxxujjMQ5PHLeGHatrdky*2 zOo!>$#lAUcg74G^mq@p1f)~qPe83lESrgM{l=R1Jt0JF6>swZ`GV+bda!Qk=EPB)J zWMTy|9gKh;!>%Ys{xMn}zb0#(s-nz*lV6LA5O`-hMw6g5*RuGeO(@i4Zw$6?GPwpq zF=9200M_uuXSnN7zna((RAmEdVW?b#)7Mx1pzbOob!zR#G@d%nIdob1`I`ppCEdhv z9fT|_|0+VoBG_~B$fz)4NgZ7FmnIXUaiV?ApJYs8*AG zs)bl{7xcR2LV+9e6tqr4swt-!nHr^Te)U)3Z<-41D`ZSU7dfI4xYZV3(Nfz%Jk!Z0 zT3xj8QRPRcPR8*G z3R>4Fhe2APiAfn|$ZdkRFt+-GY~q_}jStJxmKwy(CkpGoo2U~=Z&^|dG0_udh#3dJ zbc8qBbsSu7?b~&3NpNk!T8%!0sZEV}s1IP^ATP}9N&|sk7JokNm6LK-;H0xqF8L{*1rt1T0Cdo2OtZURT9)VE%&EUi(<5Yd0hj7On93^;BTqpig zHCQhZ%+$Xv8B2MSI}1IOu+%K%h>b#>gS0_*gh)%kE5XyZSWE~BbCfL8+I=ADcs~cV zQ7DNJp;eu+VvsAaz(Dhj% zI4}@K7a?bcmkCotx`COT5Q2B>IPDw?MjU|#Vk1$)h@pG8o#9s`ga)M55RE9GM#?rn zYLUuwX|`T5#~1FTFkvYD0g6?E15y}E3c_u4HgHvpSv+r zgH3=kR%!l8C8_j5^Fs7O8%8BCzBMBS2sGAy*c`TTxQd+@*&5!&iz!8F4k2TiozDF< zK*Dt~P4%T`%4ZjT$Q#fOE4y3H-`~}LTNezVxrUtWm;@e%8QjPL)6G$<%{s9OZ$-}F z6e;p~p@=H&pI0WafwgRi|&vnjDCgSNk@d;Aa{Ci!pr zKAj#8GOswOOPbJvQjn}ff>UW^LKs{{qpu1B+e)f15eWb67?6%V}{JR<|HpW zqo7BWq2oT&RPNWO1~p>D{a+?GyNvOA^0pGLQ~+ zP=rgFqK@KU{CKV1t7}_bpVz}$k8HPoL+smb2|)t4U8^AmQeLZI24e4+ksS!zN4X4p zMQLX22N5Oe;!W_v(Xos1PQ2PssXAJ7gHyNRfW3d_?JyNY8eF;+8g0{>l|P7@f``DZ z8K+xZI3zlzsMT=GatPmjsi#faT*ki$4XhHpY^P8iflDW)C#c3B#lyY5%Re0<-YKbG zhLieznji3_6J+qUnq`WLY4{5LQ0#N43Y@J$Ii%6&Og3D+bvY-xee8YIFyLwU)^j)D z3M(JuxhIjC7wH;;9^PWGTu1$o2zzE;wix1nq{sW5gGBVki6X{vFCDZ_ zei1#%pDA`9W_ar2#D5b@OBj+PyX+4sG)BYQ#9*X?O22=-wn4^%0Yr~M?~b?V zEsoLrZd_d_(Itgcffjr=At}5iWrQ!fai4^iI36RqzrzT~-y|QTMMsD#a-Fn$=_WL8f9S45Zv9}-8HydaCdiihv4oW+}+*XEf6HQy9IapbnbhenVoaa>`c|x zZtZS8ui+gObpQMF`(Bl!^;qXI%;0Exx9TDy`xW$)o!isREMf}w;|x%DAL?<(4tVV3 zOL_fXt{u_QZj@LNYpZpMdTc&WhwiExpg&nQ}eS2v^W)A9{ora8H}^j^>UdOY#lMo*FX8!yMCZ zhtSU#!snxu^^3>`dwI~19JZox8#r=5ZNsZ{NrY{q0Fh zlRR|D|9&l+!Vg`W-XMK7bLsy>U20YJ>S#j2{Uc-!mQeSy8WSN@D`O&_Kh>!ihK0ne zch>wwT#?aI82k!<4Kyya_taG=2ptdeze6aL)8rfQ)xE;|Zn^}lMl)mBzFh~Udrw5d zIw1Se?Ix~#%e*$-9w|Vb|B|PZ$@iCU(Y;UZ*oG=&4hi*F<`@i})KY~Kakn}gsnhDD zDnfXUU>M0Lwl_H-`9oU_Ey@0cwzwW%)t$Q2Y#2$xG1r17IGH>(Pp>kk2yT`qs(rx( z0g{#-$?Tqg?I-gTh{ZHWmW7|34q;8#l5O^& zK}iy?BWhmM(Uoh8C1mfXH;JWCscX2B)qrM_qq+PF_{Yffk3u-=-LE)7#0=M2V8 zpX%ddL+6zt*dP=$1~IW7i?q_WDXoFg(;r%XJ!&p!2e8C2U_+ey22`b*gUZq)HRS}h zUNsU*#@_f+Qx@F5rd=xrs_0+&+)(=tyZFK7>*RehN|*G@F~8q0%x!+^dqX6&Ktqqd zMI?NikseGg`Bp3M^ZXjdr3DW~+gQ{4h2?v?uaHGf)Q02RQdWyEC-`&g`O*A_(tMlu z+{4(}xkmAp?L!qyy!Y~Nehu4TcDI_q^vsKFK3C^Q6-Ne+>B)BE{|aj{5t8hWC3=}} zZsfsCG%UyXfMhQwgU@PHp3};rLzE!m5&UgCNLQYx1K#d^kb(tWo}b`u3T7r>gnho;`t-%d)Vtyd*%og}~;bb&|XlVmiZbS~eo3B&w`_ z^9SS6^Y^v+sq&T)7d5OT3SE=&YL7d{l{XHgC6o$eeQ*g|D+TsVCju3*Ff9N@OW*Xz z#_hYzRlgPa?_mK``u7qlUqmst%JPV^XR7=|?YR9O0(_)->d? z;u;0UlUyM9lAh+OfBznQT~>PQLR93WSDGUZe1xjOQf5P-LzWeg51Q!A*_a^<(eU0-NsE9~hnyZ!s6 z0m~dm0klb(4TZ#4I^AJW(C(cZ+VOk2du)+Eqr8hJ=nPA3=*-lbS`5$$h^JYbIi`Fd z5If24cfP4wfyi&HrV|ZKY$`}d3Xtw?Q&mKDpTV4h0OOo*<4xx zetaj1GA`F6IGTHt85gI=O*b)2KeVfB~aME7x`Ek&ca zeI=ucmi0Aw*PY26;tlQn)c4dRR4QGBPV=q=d+ewxuOfbd}?{tQX0ZDK<#BKk~<*v0C0l%!v21>4V;BXeA&|Vxs(7Ddd`H{ zY|K|?L~Fo(ODnqnqg}=c7bhWJsce3uxO%*EGWH=A%woSLn~RM{5?`B^#rK#l3unK- z^gAAKp-?ADo8ap)o+<`}!F9t?+S_$3h~O#hC?iAR_NXDeGcxEosyA5tYp>2Sz>dbdo+#M-9KSMLBz+Ki6?db|EfmspAL2Zr{wdNqZ4(bo}f4UO1<~9L}4Zs zi+(_GQKbEt*xqTRn`YWAvWh-UL)ui;`G77{dtMl;qmbhq<=;mec@<@2McGc2t}p#v z3->>wFDYUIn2qqlZZ|C-$Lm zrb%F7qel_C;QKFwb}B{d-mfLcDE#%<3Ks8*r$d4hVYKOVK~S4r&U~`vU|5Eq=)m6RT)2xe%dy6-g~xJ6 zNz2$l<1cGnYd(0q=4n$|H5R?ZO99%<&?G-%|Cglouq#WtEpJe))siETBzZ?Eh#6w* zCo4k56jN;u15|d{ydpU+Tv4~IgZPvFWSt-#rnpnqh=6LwE&S|{gGHW9v%&Sb^0^=# zeNJ^Oxe`rL1Ox<(7JG1A^=6SVYYWJ}dw09GGu_jRG@*^nF8r;gfa{SdEy2=K`8=S!>ZSZ2hZZT;pde|ACv zYL2Tk1c~Mjm%7!@4+#HbsXjKg0J1eN%^2$&AI&X~^y^-5!4( z9ht~adPw7p$X*cgR^rqo5NS`S6E>##C8=RJ>Cg&IYXda|plGZ>w>4#io9o$p>ztF!-1O1DaNvUbzHp{d9@1~EBy*hzo z0ifr(Zr^x~v%={(WQ`a9{-u=5!h#X1G0+rgXRY!UJ4L|>+^vxee^1?jhaQ_$`#G+B zLyF&Dk3T16e~vipgOmp{!O4QA7$hocGeguQab*R*L-u-6d|Ff4VG%;I6uJT25!aui z;HyNOOU(QZ?Cxt(YVB8~S!=mBuI>UUn}g3OZ5Q#b-e6Y38K=WSCmrK~=19#x(l4ez4O}^~-u-Uhii%%yS`wb1h zuYb2c&T8Ed)`@8~)Kpz;!be$v({7_Y+y4-#n)H`{ktetj*cdmL(Nn@CUy;&~c1xt| z%sLp--zT_>uzz0karAxiD;0SmBDD*6c!D~$P&>8WMvT8h^S@go>(x^|yqycShbKun zmu8PCOj^Ei+fkeJ_-fK3%zYx4k)5L8u0zqgNdhQ^P%n;X6Ro~hiRD< zjpQz|Ni|j@S~cuFp{Q$uAv8GbF?R87arsJ>d0*dsJ7S=2xifrn{2XtAQ&mKR{nX&XdPZ8#%(Iz+eW7rvIE6Vm zxFkWhNUO;bO@Bs7n3jCt+6$5QiaoM8tE-mLhPW>AcrvhlbGRrIHgs7zt5%(`;`3og z;9MfRVbEeI+hVN{1swyCpnYbNN-7v;lxQ#80y#Y;JTKUOc$a&b>x^mM!}_G_Z4C|R zYWcJL*fn(ocRuuN^2Sdz%4G~m9_DbQ#N=z_O)?f7xuKeLuRR^i$tCo<&$F+Z78SuI zWj-augK}DSft6$oKNZm`0cf~so7 z``N7R&7e)ohv#+MBurQP;#F{k2?RVCCLH-H3zy(?3y6w47~U zl%J7FP?dF3RCCPRFg28p(Nlid`~HURv_ham>H0Kp-epEZ8ir%KMkAPvM8T-v(dFjL z?=DigGKE4jsdfoB_u@o0?WU#;T`FN13qs6j`6f@=nrrW&VEzPelM&4T43-CKbHfOg zq(-AGbme^^-Li5r;<8&3gZ@G_RadY>A)$PO8eHfUb_{7U3738FPLxDLmSy-!qY?3d z$6mm@z^1#2`=hN|!_H;RBfh7uB7y0JCNbgcJyW_K1mcRs1oQzfta0AgF53Ba1yR|m zIAinOGjgJNg!U0)WAS!RJO`_@LyqIgxTSeh2XVaP9NH@eG^ zj7^$-A^g0KHWem161s#fdyXt42`GxuL4)*6Bz`FB7-3%pQ-gm%GN+4@qD*)u=r*9C z+%oYfhXtn14nUK*^U~%;GDHWcQ~ah#7p}8yt8ypnD<>PB)3b&44o)%r0S#8dR#Wse zG!9M+&1k+;+=2^Y;!P%zK1g?<5aTnz)`~oeq95l6c3BetX-`d!4pltjm19;($gd9v zW6miff6)`Mo&-@aQF7HW9@8vbG!-i?@R*VfJM(AlKBEefF_KpkKp`O5y^Dk28YAC2 zTL@N&(v`vuZCrbgLnpdI$P1sGG>f&Li^O){as&F)FO_)bA!!`Sl?ZzW7=cYAH z%vetO0Of~-HuVS+haAdVtDwfj5@dRk*yW-6ENlx5dv;R(0FB{#GLrLnE4oWDDyb3a zewKq6wKRu=FJMGey%p0#R(l}dRKNMkycKt$Z?bbMqQ+a_Bwp8r@APxGQ`GX$S>`Y~ zUFas&JRv;;?Cu#YnC4MN!*x*GT3oBu$s2Q6B^|!a(?^XpqvZ-Fq8e++*|EcW z)(wqC&EoyAX1RG!5WSagIQ=24(wnT^f>W_kuSJR_f#>I_#+es)HAZ#+z5@vj%XLq%`>B*>-$wRXiH#}|I9Cx zSvZ6pfoIW6ohAA;s&&diS<7a`;xUX(^vdLpPYXkgH~kG6jV=bu!>70vw&;u*yE#}b zM$prgJYOnR@0&0rCoYtlx9}=V1`?97-FTs7&=2HssZHT<|8~t)C`YR+{Ao*aP2RW%URb0wD6!T~Q{SDr6n50)P|lQE+mRF5l^7DI#Oe|;g@UvSc%*cq@Z70tB*|COh=lFM zmTZxmi%N~(yeYzwv@NjFN;RODFCn+fv?;KHJ+p!kJ<41d#R=$4-2}8^%6OgKBN2%s z<0beIV})j+37n<%JsO?`BX*zTO-h%dS!0hn_=)#{QGG=jO#DdCOc5L@me zvX75$pHa3~dU8h%ey$Zlp8*|u49n53&C5EP`gR3@OC}r#8Yg38vW3K%GM-cC0?X-wq%CIK59yzJok(rQaawD_9R!{EH@P5 z;muteu+D>5>1#7kv7or?!k|{M@?@P~?NlhX)~ zW2`PYMr2u$i80M9C8=Zj=hHaD5*GcxnFraiy8+FiU1l3hYmp`ah+S(9A{RP&hwXC^=Opu>8Y~0TWO6O`Ade4N3yH zzZwhhLwW^kcA^b*@Ka~Tnxz2R#%RR5V0ZnuQ0qN?j8j!7^_)kdOAjC+aIdtt*Po!YI z0zMd{fZUKj=jdv8vBXB^i@|Zte=8Wovn&6cOMxNadjv?Om@yl70%Br%%S7F>vU;vF z*>Bwl5s2fWS+YdyhtlK^zdkO(vB6%$gnHK~w-SYZdk}rJUz~rxCJyNGTB=PX8=eg$ z4BT=m15IAPsJbKMY7v{&oq{;Jq>^3Go!Xf5fKimu5WZc1TS0{A7gBdyc_cygViiTC z>2eLHvB#%D_$aQ$2OGQb`(4$Er%=Vzbs_bm`sZoNZ5I5sxbV-?Quk8Ab4cGP&RUO; z>PmNC9BrQuNO=gz`?uPl{-jaFBLD-x_7`cmR;-$*7w#X7fWFUx5jo>aEv zBSQi`Qv)(;P}^PfX*S(V1cb&JScP-k-d-La9!y+5A3k+}w`V^AxY1;#mp1uaK4`?; z*p)`1*{SAxvmd^!9=o0~$6w(_Q+QjxnmjCJ9np9fsgTNcSP92FOTlGN(4~s(zTQ`D zH1}*SZCDd-rnb7ubbrbHul)6b)y(VJOD!iGw^xl%nc zwXI{_Qyn#ys|;6PpjB;KA)P-4`Avq?1U9e5M0&QS7tdp+pd+!E#F-Y<4@!+_YPDao z?ZAz{>KigNHjC`SLL2*UduvxDW6~*_iK_{vim)7C39O_EjLE1rlO{gh=f>Qc_*uNH zf49ZJHoS+n75y|=55eCl9pC74pc1M$`twiL46tMza6M*J>1M+kU8ARgof!}82Yvh; zowfB_3a1QFB(vzUTxfBQrk3wihqoJ!4o5j)NxNHu2-2Ztu3>t>#;OaIVF;ZL?i>jA z&!M$zFgF6y}Mjp9ijMn>V>G-#QP8~Ur0>X>`ko`;wR zLj9E)OoJ1}u2MpabH9aNcemi1cKZ1l7!BjL&o{d5}J<-go`z zlqKN0c&-4y4+3cqL6RoTXBMOJ%SC*0^_$z1%xNnK8B~rZF}GPNR#20PM;NA8_m536}hRD35-R@yGa6fsw3nN_|JEdCf$@ zW;9+2C6ZMwoL8oq$hf9(K`kbTv&c_x_Yuc6g{qDkEBJ3D77S`gYNQypYNaA}d~L`s z->MU?tVl@2`?jr3)#J-cuc#fzZ4%P3E4dCAjCCx)LT%sQbl@KV?ZLytg(u{ZXF?r} z!*FD@4itEe1RUD`C zL=C3 zE(tEcD8%XpeuT^uWA9}mK+3727AHG+S&uyumdzjWq!h(Vfxa?gt%_jOsE%_4O~GSY zA09rTvW4y{kaKwgO{kl^ljz5*K3~D0<3h4V5X9Ny2@A_=wgH))^`NxZ+;8IaEmcG8 zo6TC+hfm0csk7jO+qIapyfGmo!v%IIxtLAq?z84Teh4|>(4cMR5IJfZ$zPC5i+^F~ zyT;(!bZRRWRM5nZ&gk2$b%|lk0P6BGc(xA=AQu>`r zoVfc`1*OtW8YQ!#`@3NukoA6JoWno}j2B#XR=ozb2^7g{z*rqMq7*ckMkUb=guKQ3 zny6R(dWBK6FZ&soBT-2EJv1Xw#nsygXfR z69dk&eOgpkB*F8#19J+2t&h3*pXTMA4w{JbPz}uSm$v zFtJx_e6?Rz-m-!%G+1GajGjY#%il!_BtlD8eBhni&mv!ucy7iQr1|+omhodm#PntV zrbu#ycW;VdZtVh^2}py{lg@?VJymKhn6UFt0}lMDULzTQc}xvBNxrpae5u&AHci1o z0W8|Het%N6Y5rztWv}HOy*aXH^Bwki9pHuphtT2MtLE8Xfrar#&XA%lAMHFNjiXY{ zL2(G#rxb#Z!Qph(AtzwMXek8G%05JT(NeQeS|`nQyecz@`TeUd|M-6~^5SIs&TlaH-d)qA-e*H%V4ca47X99fjemRSKWoO1Is+MCm|(DR0JD5v^e2hb1aRq z#d|TMcd%ls+$D;bd8i9UbRuDAnZtTi1=O=tTI)mygbw01gcM#AMayNaWRB`o+I6TV5?!%O{_jVAsBL!Bx;WG74mPP3Me4*j153(E|$fH$b_YB(iq;6A`_*b*3; zvl7(EaD87)m>W8W6~5IukKJoMY`W;A9_uv~NV`l;9NCC!kIC)4~}pp2T_a(gH{a%tDZlM8g^Q=mPrnH2Z|%`ejd z<~9rDeK*oxPxqIZN%s|_sXQw_+SqfgT;EgsoY`4u^Xh8<$_TuEDO_DycKG#%djsoU zU50jcih+(2i%o8`{n5Av(t zTwY(LM*uzoL>S^nZ|4&TpVitB=$2Gn-zu%lE2Br(r&QG2R4+wG%_|nULgIRuDKBNU z`S4ViW8}cuY;}%j z@mE{AoThhc_H_{-!g7UWoh6GvarG*$BN3<)-(J4Gs(Dl$pl3FRuQb=i$g^x38gTyX z3|+-Lt%QC6d`W{_FQz!g7E%L*y5+^>LIpF!%uk^J%_j&m( zolZ04+1B&L`4Q(bc2eC7zr>|QZWLS!$rLUFpJ%_1F z`40myg>co)n^${*Y5sAO?G zm#}LBFy8%InH$cS3qs3U?v~U+J~_ON^z>r|c*=|R_+Yni(4K?@EO!yj>!+tDdHJs_ zT++iE1>nb6BMwzeTRIx@nO!B?xxZ;A_jojsS}(cM^a|xkyQ`5g5?o|ueca{t`aC_~ z$(FM81g%;RBc7iO=iMAJ#1sfezy3VwoFR!9BaD~q+?Dp2L*rX~b3;j#_B6Vkdd#BU zS-L*Pq{+f4zQU;sRUNIkP@!}Mjz_wp-tQGOGc*Np{cf-8UK_U^EY@;)HqCOc&6|$t z*ETDur-Vq7QMW*2w2MF49U)q}gNBz46zD##XE}Gw`BoLTVE@iKMx0^QyhWbLc(T-O za#^FXY^r(npXXl^OO{3+(~E=CnzF7zrmm< z!8a;xe+OV#?{BMH9x}Jw{6jA-MJt8;cl7LW5>g^{;F(FLZn(GO?NtKHC^$42q!(5h zlc+m_!68Q4_ zxxQQuKGtq^z5ISS9##Ken}NZZe9Qp}Z8>@Vv9GL?7~njH)3Yj}V!DzF|4Vcll&Tzc z1|k99oZG~UiF}y|bE@#=tC7Lkq`O3^zvul-$qJLZi`5;`^71r#Xp$vH0uYIWTxbg| zt6m`8;H-Xfc+bYY^h)!3HC1Tkv>)ZPHR!7E48JWb*06VTxzmb#R`uc%t2%9Wj>LD} z^Q<;1*zSlaSS!&uL#It$o9@N@%)Orv^H~d@OGKqLSoZMVDL~C!vr@#}r%CGkH21yP*?KDNOecoz7&}so11tg9fXftFi?tivKb#?DvKeixCM*9()$;GNO{8j-p=RssJGGq z01mQmrtu1H_=5I8qm)b@L5b%)o$p0i`-3GX65BGr2fJCUhs;+USf6e?`KwTo?eFCK z8^oruZtnOWOJ%aXQ1k4z>mE45Qm$Z^Rm&UYm*jIv|U2V*VaVNt8_9`pm;KNX6L?cV2Y`k8K%OG zcN)W?59kO{S`D<+tdDc6>$!D~KOWxXng}%Pk^D;o#vkD5@yq{)2(U4HS&n z+&P};M1FZPC>?{BIZA?*nK|b%$FMpsySZ$?DlSnr#Tdqr?b}K?3qIdGyL1?1_*Ift z4~r0QE%)`_T_i<)3Zlwxi^$Riav8K~1tw(i83k;P(83M%36Gh?0Uuu|ugcO@F@y3?a%kw&r% zw5xIqnKt;8l9^GnSZN>T`k6Kgn*&6aTSQe#BYt0HuhckUa$;a~@Mu%OdeXlS$8x-* zp~st(u7F&vkNjpU&;OSPz}w?@Jm5r*U;8GpgKAuEK&c_gWaWU4@`L0tPu^#;_Au&f z@R0|99unITwrNyCL84Jf|Fi$MqSB|rEsE#+3YOeB3Lk5vauf9mOJKtxwH3BiYs8qO z5q(0+ed4?-xMPpy`5xn0fW3`Ic8J_6rqrpa{Ueb&QQ$m5;#_%(*|ZU_5p)jWRhBcL zSe1ymHH0C})8_?6hD`W+|ilWxB%gl&o{+eoK6?mPi%(9z& z`3V8P`|oycz^rG@<+rtKz=vV)I3Sn*$^UZymi-Cv#qXV7`;+y-Q9oibYVYae6+7=o zx*vu(-cK&#k51^M#9D?=7^cyKCa7K^%{#b*@j%A28d-r|O{3`RF8)IcJSY9I$$+2gy2DKE^yT|~VevcMHXnyIDEvAb; zu>qE@yfaAzg&f1s<_>THrlI~tCchLUVjm9Xw!RNmFUOaQwlUCCdmn3}B)%6vH-UAUyFs1*O0+`}|0+@dSn12G8e*&0)0+|0H05b%rtW)p*jH31sOKf4D;uClO7l6q{ zIKA>GASB=g3Yg*l4FwGGar3_j7$xDe(o)W{KLTbrM_mB0bkRu>L(dNgq6A(keJuCy z488)NkJy}0F|X^l?VcS8_P%{aTxXOJ=sAgPG?JDfwt&tTXY&{z5QS1!ZwjAt+-eFI z3NG@KA@In_AV8s`@fCv7J<_z~qf|@?*P6@E$~(w!O2I+}Jh8wsY7&3DZP97CAMo&r zHZ_VCKRQ3z+1cse{{4L0|8aZt`p$%>EPb-a@B2W;>+{8I7J`p;c_~xqae6!Ah`oT7 z8BhJ@7fZs5wsTJ9d4f_5+ernq&w!DJ$`cB-K5LTZpou5sU6&}X%`lhn zUg)v6p|g@mCr|3kYvaN>RS&fd{4zJ_!(nL0;y}HGLH+QoX&cXpKo4#;Z^~09@YzPP zD_J&kMC(q52v9{p9u4tQ>AE zoV-)rJy~abw$$E=LZi942s@^tJp?_721|Lya!hMiF2)lF1;UQ{=zyE_C*1(iPmlHR zW(f$hs)`%*s&|^nU!TLqs>Z;e0Tb2~su@jCvmuDT^FU|ipSFz4Uu_xk zV6&lunNyXq>d%~kFm=`c)|SCAIW6;!)Caa@RK?*(vwWgqWBTfGyS-&c8dsDi688(Cn z5#VNUtTxFkyz!+KL{^+7vW~_nFQHZK531>en0S^L`B6Ozk_uMLJvnUG-@Uydq()cq z2i2&2GN!?xMM_!BG{o}S#)+l{)g8JYINB+KmJIzFea*>&L2Y+v$Z^&h*_T_8M>osV z5G+DY5Y5C+o`uO>3bmo+ukNUXqv8JzicA9K8Z%ZzxKpYd=f-?g@g=WI;EEt(NKtXi zi2+MfmIW*lePWywD{0>%`V1D5BBH#N@XlS^qb*lSJrnPBg-e#g6((yZFSeXMJ_upxmis)!3N)?pW7DH19AgWR6og84< z@76v$2PGMx(|6$S+oa3&#Bh#@64wKw8ln2%Ge+%-*$4e~b?2mngQ`IrW;eML0k~2iAP+39YzNFf4_!Biu7vbAnF) z&?ESFS_XVhH9j=!52~Rd_=9S+|3)>M;~79yqlk4^yoA=LaoW#nwZsO}E{3z&+>9+` zqB@lYH?T+J?-h`$7fE`nO+fK^Cw$94Ju#w@h@SkbDb^v2Zh!ArjKmG}>*1!RucPXU z?JHFfj*k3!&*5-*kaw*J&AKHj!uQxY4SV59K{e?ASNp!Ni?}r-&RKI>Tu@6Mg`-yZp~c*_9>t*d&v~s^y&JC zwC#aBZ@rLVuRwGupG$Zt@Zx{9*GPW5&ZST)LIrYxen%&kHzg{fmRo#EA+zo?{Bl_h z5$}0?@6V&XU+aitx*VNlXr;zbTH`#%qR`y+Fz( z_mWI>a1o17k|MsA3vVM#5Af`-7842lQ zWII0o8E*Q|TTQDf%mN^(bmw(V3BJp!FT}o5GFIv1#5q*0&5F6gv^#lLj4^sNEX~hk zYmUXCzu9MBlia#aBtK&b+_alFwSi#nxFV;iKz8BigW1vX5LbVWY(DwS`MZjASxtWY zvxHy_HQQTe0XE0El_^r_>B3Y@eshbWB>oMmYBa0NR^)_OSAIC?;JoAu-TtllMz_`U zR{r8ua>sDE${$uUpRlvUVO?4Qb|aVCq|^>#0e1*1gA-p>mO;J%Mx|`EB3+RN&rqai zlg?@~c*xeBbHQTaGQ@$i=f1W!@t(v}dyhI(S(^?(zgP4OO$(G*UmUcrfU4dYW#JODnDM2Ry$r{EQ7nb z#2^Ev1g8_)z*&zIM7~Z&C!usn_fJ`7|4&(FJ&rJliiXOVP@tnbC6u}nw1>fQ_=<4P zf}N;os^ju>%);n_9Qr2fD)!(nRx>^hWHmhQGSYhwVrQhajmF}=o#z?M@xK2k%ghK) zT|CI;cC>ppaA9U291O5TgVXXm;!f){7@t^Y*Z5U?v_LcI&5KQcYrmgVMkr=rznH{V zL`rNV4ex2XIG+hI&}ayGRg|Na{7iX?treAWumfcR3Dskp9t$%;0aP_iZ5G)3UZky_ z?j1{$t=7zoc}_r8W22qxcWR$II|~C;HQuEWbbnM$Oa?@aHQ%pQ+ccR=xbN)m-?LF- zN+$n=W#}>gH7uirl|=GaSjL6%WS8Nuu#7sypRf$K9w#Ep9Pu)6BdM;@|J4WZ!@Euk z+UG-FQKpw#T0k&iW9#gidKxx_M$#gd4P?1i- zOYJ4++Bx;TiVPIbIi;Cb7gb9H@_slcOgnRsT(#DF%s)V~t_S%Ojss22!XH!9#AQzU zm#HydIEJo4^toybU2J541+!c*+-VPK2UE*2E01=1e!ipEFQvV$>R8>~?8Izx5oqED z{&KGu;lq0#o$}dy{dL>L6TjCxYKc~gi71_-DJ*{yJWFS3J5GZq{N3EjHhl$;zzzCz z@RRJnbY(;*|L)2dukkknyE2zef4VZu^auaim2uD&G6|~V&tb~uw%)UeNVygIAt^~; z2(8&;l1_m4qF;BX_l)p)PBvy%;QMd`NU zr69dj;@n&u7P*W|{y5ww_r^YMe%Or5}>Z_s7yEz5%B^pU2jb0|5#CaE*d3m>e=g4N4b<6uyK;fFh6ZrB_kkk?!Xc%Z)cReTDvPe#c*rdGRaDDK8I z$PbOfBSNd|BNNjAN)a;JZ=$iFXgIu$L`8C=!8%{g|J|V7i7p~J0{M@r@ln|(0-BnW zwY5K{Ciw|X#0qyRXfNs}uR#O@+dy)FD76T-mSy+WFgPs$ zC++7|eUya-4U<4kZ8ry%OnAiafkTrVva%qlSva@y_p))>NRet*ozL{A?{>x}YsRAB zinb@!wIDv*Pcgnh%x|Z_tjwF|!fF*91C2)|rs!!D&0*1|KHH)LT{e+9PJVpQ5sEo&UWeXt#SYe``@y?lh z4Jk{Q=CJxY-g4;`zu)8G#aWyH!TawQ=SE`Q7w?-j|DF$1OV<4~rd>tj zuhp*i|1zxP_iSR0Qkp;n99EL&4KK(*lKdvSPLaGH$~0rjTf_0y5($}+d@zS2F3>&O z(kkjaM@;`r>&$Yy%$C8nA+aIR!=IA_p7QuQA^&&p_nbL0Q&VRyruGf@O;=Z~wSLd~ekzE>jKulDYPuT-4A0b@HlW>ouRQ!< zk>lQb;JO4XMBi9S%hxJR*3J#kn`Hr}>Hmq!XGz=#kY)s#6 zl>Rl9o&ht&&po#n?YThho&i=weCw+TqugK8cTPL#z8R>qel)-l)R*SY?ZC49<@mE9 zTvTrdk|Q;jVl0XPc!pHXgu)t${X!oIpRM}rRBbV_B3k$zNcx!=a9}p!Cb~L zuKhXRv4z%V3GeMIlU4H5KeC6*cAh2lWBm74F7o9nwF43^=c()AjiKW>>%%C$`FGVM zjp>yN#-6b{jdKeR7tl8W3=+5>=|}=;zg9z?R3-W3=GPb$F(~}}MQm2Ox5`QWTOvYO zp2OQ1OE)WAm!149flUL>4LWxee(f7ONcQtyIicR*9}5hDjP9-uG0Av(?`ISUyzm3A z%uC;&2?=@n0!{xVDmBK0VxQcbYJEo{_j_~~qQI1^>2>4-iAv!Ah)NpG{9{f%?un&T z-c6$gffLiB#e|xl)Pja2&nhI$LskxvJHOsl6zx~hwb4TsFG8(^w8aa*Oa59!Qz3PD zK;>enHlaNn4M3)F%rSzNt=facZaYbK_q}r!OFkMBJFFN>d27lXC9Hr9wAZFg#cNY$ zqV{i7CUc7lcf#6zvGovLPE+wJ~l>suEw zeo$N+^y&^~S&_}caxV$#A>GSCi)?_u`o7z2SS>Pnntf6B@a)(-mk!oy3js26{esMo zrIXWgW&F%3#!ksV?a&G9eil^+fvlpbKN&K~PcDf91$kx>bf1SZVI=skwHKk%Ek)FR z;#3&yUmP+pCc_QAn@7yc5mEX`oh$F2bSDxPJ-3u1p`jE z+-P3%8V!Dddu{?l2SihWBmxhCr+Zgaf9EuYuXCE0TX2e+!`rj>f9EtgP%dKYp6c|J z#+nd}>a8dNuRdi7u8x`~O(dlpXaA3bz{C5u zJq-GI41B}cA zi%bviTqHt;xLF!{cwev)8>9Y^JUQ2%>pnl_bqo324A=$YR11z=UiAd>_jiX|w^PRbt3@S}U@X_8A5qxiKbGo&Q{1E*y+tp)2DnUA6Y@f1Z$&U(5 zja1&lP3*InrWW&Vu6q4jQ6_^ip$zJ^D5C=Gul+R^Z-X!Ofy-lZOo3fY*1T))9L%Qw z2N<&UDlhG|`j{~-EW~^jZs5)-0cxScXv!!pup=!#fGnxlL|W5LYq?zV~cinlZVfH zhkqKK0$jUj|1BpIp2BI|@=s1C4=YXa(?2;GK2T0(+8UITd7vwO&B@H0rC@XHiV`nq zV4y8{qt|V^R95i8T#=Tlur9~9j}y32rI7Jmh8VewvFesZSg@6T*tqWj-Wvn-8#*38 zXwA)@)?6`pf1ph){KJ@$h--_dz!Dnyo6m&Ogz=x8479dHr4Gr&^dP9?tWB8vl(2_H zRse+?HISHIz3h3UqC86enc+86gLgri?YmEM)p)&IkH|3`ml15*RC_8P8pLa@$Jxed zGNBIzWKy8vF$szdOjKfjyYR)xGw$T7l=lYCz+zfUi1D&C$Y+b05m#6pC7m~T^GNP0 z?L%wKhO?cIL;ZqMcWyz>xJdxx!QSQhRmo(U_?`XQS7;=y{}WmsUVA>3>7GERh6iMj zec~>hTV@GTMY3}FHusax_VKLUrXvPEYfZgbGHc96t9QEVCFD|FT>1o3^|WskwhLH_ z#P@{oPF8WshQYR731Q?ES@(yMr*H%_B(cOx61k-F1nps~A##ms4(qVx z@K|WqqyXh{I6xXlf($+gR0@Fvv2291>`|B>?9!H7R^6I;$4{ydMzlda86U;pl@jPY zGpXS#n^j12w6x?Apq@X zpQ9It!NID*9&v#3J`6eUT*+xj-|u2un4%Aay19xQ1tM5_i;cy*i3_}tZ3DmD<8tCL zM1N3lgfm#HSgK6ZouQ-T5jjkk+LZ#)O=A~qoHVV3&=cmRpZ@k_*u}T^1P=K`tUA%e zRMPNcQx-O&49qqXx@RpPT2jn;Bp#LX0tJVuxNN`3=P#g|M5E&xyNfuD{%{|GSQL(~ z5SLedma|{6Eo&LEjW;C>xVzW=86-3(tlV(*!npan5J-;*IIoKmUq`+CkteBdsZpR9 zs9RBE?b^0(L{@Y3AO~jEZf5&rR`C(nb#SprrdmHJXdo(E(L(C(RaC0rs18F#?*q4q z%EuiC%IH)7o2Vq#dzV1C)jmzFNt%U*!-EUZQ^q1wBLxISi%_kFP(JEeLYeX+cTj3K zN4Ku(z%%4T-S(J%j1w6Gpzb$lnkNeo(SMV06tY?;-LbeQ4DoL&apbtvBcl%0&Jk)z zJgO;@>+BZ%QIKpW{d)x)ZEX{!=zoez#2GDOk7+;$dQa_EsByVlST* zgNTg-8NRW_Q4|5o3Xr6%ZZIi>WWTwyo18}Pj!PqrmzT+0$u67AGIhjR=7q=>VB0q?7349eiSp@IO zWx5#FcuJ`p?PCv=zSdf^=__*tI_ec>R=IkFw&RRx<64br@Ox%R{5zv9oph=6fon^0 zY{h`ze+X#g@Mu1J;E^U~<4djQe7cdGz~rib7X-z>Xqm$yuzUkgrtJ`pYjcy#O+her z7)}%=t7NT*m0o0*>r-Eu?=+o_I9nlP)=x%|+>2FX6J}JqCSpa$&X4nuR@YdMe4;8TX7!E_lV(G@Jhi&iznx0tct8E(!g=XgeN%fD>uIOYr*hW9y|Ef~;qok!3vTP&Uxh1h3 zwOc}-ua+UW>+Rs;Uzmn|0OL&At_Fa(JE$toZ(9eHv9=MugV1ofjikuTNe;*A*Q?y? zYAp&YbAU!Sgc{|#8P_Pz%u?_*iuyu`ZO4$sm@z~5Fpc_1 z5Uu%phP&y@1hg~&h%>P#+&?SjjJ!jlNB3^W24CBR$K*YtATm-fjr1&PV)AZTfDvmM zRJ?FDwBnua%_z&fhu+E+lGCR+l8S|TY-wp>Ne{ztgg$7hnOrlWIqDm;b2F15*N!d; zuoL|ih4J~8;?-8f!`!=orh_jQ#xGZUzr3Ry!ogAYhBhu{t2s2=E(7&me`_q!ccB4%h0 zQOFH~90ydgDt{vNqt(W`2E0|BU2Uaw964OW#mU!-kqV*1aI+S*^-juSOydJ&VJusizmIvM`}3nrU_;nHnYMJ&biZ^iPd>Ex56{1$M+<#0|Nsicc9xN zkqY$WlW8(aq;b+QX7dG|oSV{KBk93#bMhr>G)FfoOkexJ#}uUJXYs&I` zfsXnemi7{)wO{^JlFGht03tVA3CF;weEFH4VH2un8zWaXe-njAuMYf6RucW99fgKO&*s?Dj;#_EoZ#m0B~Ih{2sNdtAvf291iD5GY)`C62*osl)CEL8glA@%)I zL$YRqM?ABu6a5MM1s43)(nm)V&{nJ9Xww8@iZ+l*R0?{H;N#R0DLpO&S z&(H6}y7B&0wzvF3>1rLYN|=wD9&|S7`!FG4?lh&#+$fE})gY&>2YYVXqq)W?bekj=bZDnL{+HFN_Z=A3tl?8=Wte0Qckbt5~AKDl* z>K-vnhPG5!Q2o6^+4ic3tClm@HR$#v-$DYv@5o!V{yfH;-cMB*oJ{;dN0ZY7WYQ?c zA1k3xM|A?h+%)z1$(b`)|3ua_QptPwVMtn<`d@^$X;|=+M;Sk_@F*{$ zD(2}oPO(!|@fF0=n(!?3T4}?(;?(AhvmWtQ+(fk0}dh}1u|u*#WGn4S_0fQnupeU+7Nob`@hWu=(SobuM(B%;u< zXFTw#ckr#66xHus*V36?bbFZ;{_KT1PCC z+zde4WCL8N!j3G(5;4Axg(h~b$&I$I<}0nFqSXMRm5wu2FLoK2W%-mb%F;ssGBOhS zx%H^2Rc$CSaK+){CJuPL8#(8v^m^El*gyMNLiWwz3-Ig98u_qYk9*_uA?z9MYpBW1 zppd$?TA;J|j=pSXOyYQdQXOEUA=we5V70FV09@+=8{uOoq}AnjmqVS6FY#rxHWhd* z1?`szpB(!tL?r@quQuFE>8)~mQAibZCsRwfq>p3R$^nQLpoa!%bsz=19%VKusC7x28<>I!gs&nM_In|XYY9<&~^x!FA;du_@P;eA_ljZmP#}%g@PhHKysQ+taV{SGbat^ttpc%@f7~A zrp$^TOoueN3#ciRgD9o-)|7F5ZOZTx>^$@b%bHcNyf$UN;15A(_k>=52qVt4 z;BxuAx1)pfXE-wi%;@TK0h!ri_FnGQzFro2S!*#aln=0vjGj~b+DcAUvpMtmf?h%Jj#US%bn@W65$ZW~RcegG3s!7AiMW8o_6 zRxo$S1P=$sxCw7YJK-ihoJ^+qmCP6T=`KVcWEL+(AL!6L^W!?84h~Xiy4V~vq33OXn=;#@Pj5|`?boKvHlvI@ygGhFyMp#POKP~e zr=%Yn6Vv>KriLSTrKmv;s(sXh)|kFf1@A zWDIO9vy=VZtVdBBomWkA`yp~=r=|wUq1Am26@@Zd2~>eCY>ZVA*IR68ySGoeLqL6t zfES(;StZ2%jC+EbO%&i+-&TMi@c5;1VB|sh7Ua`=I}(=7lVJt@LGVc@SFtQ4tDI4D zPo;9~HuJyLf*cjNe(1?%x?JXoZ?KaBHD#EL6#|!&H|P;5tX4#^G^NzfBQNFQLR3_w zR@}T&-LlP>x9tRjG>R`|*=yn>=dQ>4>EidJKT4a&9Z~Rg_@jJH*laONAznz5-qS-( z-Krqgd66z;0fs11b23T~KC|&-f}Q`@{^fts{^jhy_AmdnfBB!-zuY_pHk)AWi%yr+ zL>3T%qB5Y4)h$ra*XK?Lcw+Ql1DJncCC~t7H|)O#F#nwcn6t0I>U7L92H=bdPuXE0 z=*)RQ`WbXOidqDO_MUG03oB)!f?%aT1jemi(Ik5lFwd{BQu9BsQUX3mhmw?yMR$=b zOZxT-E4@7Qx$fy;&|y+Epw!~tm-UK97)8vtwi+Ihq0n8>MJ#Z~1Ukl#;vRj&9!((& z!wD;27%z;yc#X^XZ92)Oz7!F5UQmJNsS3B*5U!RYH4nl>|VpQtF$l zq?-B}>#wV{@aihHj6@;)b(QY^x=MG8zT}tpS7pt*_y!v1S}aiuxSEr)T|L$nw>U6t z#)-p=u!WXa`EB2c8+`|_sC%RiCusN!Ee&VxkRdcXy^}lr8r)nT<^!KEF97qcf*<@4 zo0nrq4(6{!hh~h>B%w)eOXoeiEJ&|}NRxe@c%fC$- zyLX_b%tXb_Yf}ag=M(+pvZ9xMLOP>_Pko%Ib^5zRcOFB~vmFcX+{WyLX=&*Ru zS)&Ps*@xo71AuD_8O$fYO5V$cC=(=%Mc6c<*d0cu!&0>o#7w(@Nwu1#cNz5$==Qwk zJa|Y)UaLt;s1p!VkFWtMI&J`}jXk0+T4qkP0s-vP@^Attm-UK=dp-TH_2$dN9L@43i}FC64Vcv-xnP zEE1s=9;0G8rj*SEW^ zK)>gSIm=NN)4`(QOugfy*2cU5A;Ey>%du%nB?{41Ftu{bJ&4y81z-BNb3J$GAqJuZo2*Npv1B zf6gA}8TZWFa8EMj5(th?DY}JkeN30!-P|;>dDQx@smUMK8Q}qD;BOV#@vG!lH{#Ks z{gXQs0z9YR%-c_8n8#BI=|;Jk#wC%OgX}Z8n!n55&DddM@-91I{j7;;HkvJ5b9^Bp z86`)>e^r&}3%^m=lh;NR&4*7iGy7fJ*$u6 zFfXiy6FysOf&QI)Q#CE$ezKNly_H~f=~Ja(XG+FjRY_|Z=~(VG;PpTh*LA?+3S3jadUdL3N^o1rl{)`-0 zt%qJG|Kj`7X#_koxuXUCoB@dL3gf>lM;)vYPW%!nHiDg@W(xMn+?TDtZZ6tiE&VC- z-QaQ4a+1)6Pr*Rmugl|KJ(&|Si*%3MzdaeGe|j>FZ#@~V`FDZ1wH$>d6t;>^O#zd7 zHSGRBnarl-)Q?AS7apt848NE7Jr!iOU^F#~9l3{mbB(WIb@%m+%1yVMkE+AOe_@>3 z^;b2wz_A>Q>f|3*A)CbhNKRvaJYJRKwrA1$+0Tg%2>*-1#~m&NwB|YYetsU=+Fa=? zdIC)xJGy|Z^WVAhf;s0Fbcgz4{yecVSi_)qKl9}!!&7OSyeB%u>j7J{8N*u{53E?t z-EH0YHZq?gptt+I#{xjAmQW!_*SlTlsY%Y>_xl>gNZoHy)olsO;1l?qZHT?-`fy&n z%=|2-X3@v%ls4w_J%*jZs|fV>O0^-BOP9e2>wiYS3HtFC$lRC}d|Wgiirzjo(vkS7)r)fP#KQw3m1ZIaMxI`8*f25{BxOA?vQu&?Czt`e zE*b5xhTFCQTp7kBHEYMo;GY< z$)L9)&g%_hk~d(-{3|Cz{+g2sXaO&g<^Sh*s%j>pCBU-<6=*CQV7LpgmD;pk?~VSDIR-axzJ8CuDC3 zSS6gn<&9Z{w3o}qI*drmXB+aj8XwB$qf;3ab={DouRBzGNHg8;eO%* zKYBZa7ql+xJkVtbLd;s-iKJOUUv&LEj`(Ar8 zn$y6;<05V0&OcVOOaPq#P){aO`F83`*zd*u&I`CP@g^#{B$z9zc=tdfRo+k3ZcLbs zl6ZhbrLv8lsENZTkf@|9uETv~dSWLfFSXyUn?Z1-S~)k4Cx=m-Y*d`x`X(x2;Ju1U zw?qJBxste7QHicD1SBfWy$mf(@D3cyPcPS^E<(B)iiW0ezlutt6-J>7HU=U=zql#N zY2l}FqXTTpRI*)snRJ@nz3z(5Ysk@4Lk ze>)q~fOXYy0&kXLD0v}WJZ17t`T%4x{0B%*0~9k$}NAOgKS(t65bK#9tlJYRidng2vO7B6Wl5fJ5 z5`{qk8uSKT8PB}x&?UssHTBaEggNh8B2HC^ZiDL#*P#+YABjpSC-j)O9Y&!*WUUipbFH|Mu9=iylZN0;xf zroyecU)?r*hK_k8ahl9q1g$rEP?`feJ5yAOmDQR|g7sEWmlaU~NcM(|R976~O@`)( z^%=b1r?;E@BolEkis&;$HDF#qkUo@?;12-400MKJ#!gQYb!DYHI;LB9;|m00RvQh3 zjLJ+|yO4+*j)Rhe>KWsL*)M8ja1GF_&HNepF-y%c3_$b~e>z847={5Uim+ab(MGwr zAX%(^O}^ zw|F&`P=3Uvx=BosD#cZ+8m-yREbh~`HqsLs3qmU$a@E&jNNx%!<5-3^vda|ZN8uS^ zj+j~I>vk^EU%(I2M^K@gMw)*JU=fpL*K)zVUoOIsJEyr zSS7CHig3Bx0$902KF9zC#-<+|gBoLBDj9PI2FOudghxLF;y7g0hjTw$lz%H6+j8{{ zZ>u+%C7cgR$ z4FWq)ICDZPS@QB@*#SoTeU3|LT*jUo+@IO>Z&Rao;h9KAE)SzZ6&(pU(6|Zvy>JNG zQdk1r{VULOH(_!5VwJ=BopUKr|As0VtRm3iqJ8-KFH~tAgeukRj)PDoS`ez_SJUmt z+fKH#A%vL26*NCnWh%pFk44JlW+;T@3PP1+OG356N{A>gP*kfFgoUY3j^Gh^IkD%v zzRZQ#(xBgx=jG9`eO_{)BQ72D!iY?f^J9KPl>oB%R};nTi13W;0jG0}DdY)vh8)O2 zy_5Jd7?E{r)^putJGFVl0fxqrqv4~=g=S{R_|h#Z{Yu=b@52xp7aAMNe1BhAU1Xye zK^`2CNLOr*5u=ONf7URYhFC$)xSrWp|wgfPl3Yq zvrCq6Zmm$*$}vKW(Sq*t$zo;SY$uHMMIW51Z*V_{FUGJCF{Py2tu`v9?dMFvb1|x3 z2rf+nlOWAY@^~u>D4%E%2HepQfl4;8&#cz z8uz22nviZ@J7$T>hbZ_+YY7PTwy)?uU5=2sC8NJ}4%GXnhq;@a*mfudrpWsS5eW)s z?Z{+x#=R~bVFgWa7~8B1R&XKMvohRNE>(ZFKN7Fhfnk7EdmzFDE1P>AZp|n{#wuhWD~>&3b#x|Qa9)41;!&b0z(PZA(}o~vQu)U78xBomF+pT zc7{G*SzL*)eb9O}j0dqdHJsg<`qe%^ca{yumnk`as(t-WRf+ypRVw*tb@8eynZK$^ z`Zpj|Nu9vRBdJ|RJo3nDPZR^gpXxMBbQ~W-q6?uL!qcqkXEp%^tQ{`?yOpo^`E0m+ zjRIc{DwJ{h&2ZF)L(Iq;0HPT3HIb#e%`)CJkRYvT@CZt zZWlZ>t5~e!SF^l2g$j2LUS3Xt9>J_^j%!wzt$nI30)fNyV|2uN|VvHuP=4X_s z&671eaN!j*?v0pwy?V%msvade9s|x6%4$-EH88y&Y1J^S&I~u2M5z!D*|`em7Op5R zkFG!{MDPG=oty@dXj6zs3HGJ}p9KrI#sR@gh}=>BtiV4-$0hwZ9m3fr z(o(HFYq#bszrt=IpEom2_sS|wkj|3xtdg)g=%=NqfE7myEPm2%;jnGP`&NT%>}T|~Ji)wFQH-&iGzH&#gx z#3}`GSm2bRA9LUp!;azz{g7mpUtnj7rZ~G#Uv?VB(nwKSt?(92b466^82wl9NGWhgW;MLK z_q{cS#$?{}Uqbm7Do4(c91Cwufbud?_MRwYrYbP99%Mzs z+f~BQfB49mDKYE0gPlJ!Wc~Q?xRzmla7OX~=_JxkCJsDI0=liOa($qH@V;p>hL*j| z;}wSL*^_~xU0Jg^6n3_c{~|aEV|7NWut2&6HG4p-EE+N`u_ZeBNHkc4`yd}DW>KBGKo_MU zDTu2EWqZg*v#eYF*P1GWcx|&yVPaubax|O#N?d}JKhhq^alg$ps`)BWQmB3^e8(D+ zUaF>TR(ZFqt?MO=h?3K2F<+hDJ<_YgEf;!_BP1*!Ab>|4@OyI!w}-(g3yY&tNETWAzCm&qnvQM()d-Lr>Sx$5KbN$G4vRbdpgDU7 z0%|z928kqjC@xiX}I>OEDV z0190v;lF7WPDkKgGc(Nupv=s1%g3w-Jc2b8D zwyLTMD6hIsP$0d(nIEV%9ej_;t}H2{#Q3ADvuIE*K*(R8qi>>`YRcMth^YzBd^*z= zC#;F;w9Qpnq<@s@uv4hiJY5pcoKTs-i76>qQ_KAVK`-!=c6bD+Gh;!uCSK8#fRC;G zvYk?VMHy_%>y;2D(X@#F$UYWxoDXuA$_Q~HAa|$W&uR)kx#-LiJ((RaC!uAdSMpV= zlP#mH!_oJCOL=paM7HeQ^^@w0PU!+b&XP7h7h&Ti8#gD!NQ9g)DeU9>&vk8noXrve zf^fH>hic}~(_tT$2EI=9o#9jwN?b4V0u4PY+?L>fgY_>A3Fzv%HPzy!`gLz=ED~k? zuJY-k;7LE>f|$Fms`2~m0%)=eMbn4+tbx}uN{38NjRR@U2<-!x5k@BB;t&6w3JVYF@*7; z)`JW5nMUB;f3HI{VU5$yRg8v9{nERMkss$$h1`3h_&~e-0ZcvwBX`fNb<^nll#&5Q zv(KCgUT5QdpH6o$Tm^2%{?(Pi}yQ` zjSAe{ja58!u!3s_5ir%vuBqZA&jNSz5-P62lb(ZNhi2L(rYR-Uo=P9BzgEZIW__1b zv0-6KcKNH?-Q(3+V&Ne%aY~;Nh;1L1q+p4jiOtvX0=TvCz|N$Afv>xXOVcr`^j{N2GkdAz9);n1o?=$u@-+(~4&vj@E(*P=oy;vL?-7v&^p^mTGO+OwXR1;sF zC0q$@Q3BV97?#I!FT7}8C;L9fc-ko*3o5JCBt$9G?-_dPU?0k1&&qV?!FF+Myq1?N zq0qJ1QrY2BE1wRb^?c|OCY`)9k-{hffTnWH(;-c1BA-k%Cq6<0*>z<+v`sUa@jN@c%aFu=;9O<r)K)48;IH_&gZLqbe6*#nqidg{^SJ5N;4-fc1o z$YODW_Ph&3MY%n`56mM%`6BdB5{8ghrBw1Y34_HX**htr{x=Cj=>Inf;|_QIe@eob z{of~H$O_^b0RNeUsh8PF`zHw#u1+w8d6+Qz?@<_udmn)UgJV-QgGI;*pv?82@%OQ? zh`p#bVM{9ru$Sg0xwxpuhh^`iHtH6=-ZI--+rQRQy4(Hwv*iIQYG<&1WoLnA+!c^8 zW3GB+gtydU^Rw1x`hCRSlyJdx?uN5(a>6AnHnPb4yn~o~E!;<`)>Hx-C)Km^l3%&= z%=v2;&_EX|fpZD(iEO}jq=jzdaBw%tJDImrj)JszVteB)Xo}rM>bdZ`WLyvI@v(jkz~3-^1e>d>yf z*Kc~LR%`8b7NFeMBuz;`YHiWw*QCnyY$(nehZg<7YOFKYYc4s(SU@e3r8c3Z-evDX zc4Ik^L-vll^zg<$L$rvZWfrez?U&YR-nw^np0knv&{J*ZOill+cCsClf+beg$znFA zT%Ty@TnnEWhI;%}eqPR?THk*1WeBfzh5^F)y>&7wZA z=baem@rv3ntZly5&t1v~m?sNH-6lp!0J{Zvdpm1Zs60*<0_)c6>zcZiLcVi_xZUK3l^{nFL0dURPkaiRmm8>Ij@Ls#!%Dc(;`L8hy&E?5k0|Y932o#d1jhrz(u*e`giu z$Nx(eX8PZ%FysG|RhW1G167!`|DGz$>i>5YX7>M5g{k}ZD$JMvrV7(K{PDk8#T=pk zKUQJB_w_ahrmwK{G1f=Tp{khQE4}pS*&|$$LTm4sT7%v=wVKIWdV#Vb&zjZ5$Jvg#A(ADN_)K;{PTX{p_3Zk(Cb7n2Cz^D_$k zqwH7$!|w+%H~@R7zNWKQKpcT!-%aqDKj`I$C5vFvcs@pfR@RK(@p;Bis?5T9-FwF$ zExs6P>#zIXhNPcHasz^yQuCUM$j%J9BIDtGzj(sBKHc)fZ(nbVmb1$r(B+$BJ7WL=Jm;DD@vR>hT!SXb9k&y&_<5ZC~CqU00xmlM@*KWeN*t)Bu$j+KP0 z%T8)C8;jgd6eCRB<1^rUs+XRnjQp^tQSEkR@|k@hqy|XQFC!B&NcOA#!Il{EuD#50 zN-5Y?l)WwcO3``!{`@}Vh)jYXc*B-BqXOhohiuaO)NzpREB+9>^bRDT9VkJNaZS(q+?Qa3&6IU^H7 zpy(G4<~M7pXdh%Pm3Iv{1W? zrIv19sihOP!jQkzlF(mjDd6Tyl;4Yht%DW}E_UZ4BKeIZwwu}+OZHJ$xaj0^k;_eR z*@^%ybJNIg;)~9)TGfZkv4k?+9~=RpBG^twlq`9_U#TVI9>1siYlj)U4Y8BvZQO@z zKb4C{TnW{_k>|r}Lnr*vPE?s<;vNhmZb@a{o-bj8w91BpSAwN#ZOsA?;RvmghpRAE zri%@>f{U_5PrOA&8sMj2UBd3XxU#LKis zzsTo@szs{~F%hz%0)19WEq+)rg5MLD8$XujiL)CvZcrJ1{CLG&-o>zVb9h7qL+oT3 z_r2kCRhzSGE|Jew)Mc9M1-zEFo1r1=%WjBC0G+9=gI}VqoDP1P+kO%6j*G@QvoJ{iWMRfPidq*po?0F`G-l+Q z2vE{bZ@!}2XqekYyN(GiO?+U$SIe<#Yu6Bj~wGn zP>QSf<}8I*3Bdg0EUn>%Q`ks+dGdIAxc)hE_N&wX`C_Dq#3;~_J4`xIIpA?-xrfmq z;t4}az&5VDxh8=B@enY-vkdw&6}CAsiji4wqhVD!F(Kf;N3VsYtfLLL$BisTfNom0 zOU!@F@2j5Ia40$?|7<+4Gqe*#CTyN6QPfh?!ZA1HkKloz! zlzcjoum0C>>%+xpkh7#C$#}J{JCyM4q7!ua?DkV?_NGGT44diO$^OPgFym!XUyv4T z4S5BBKlP>0ib!Q+)nWm^&?@C0jjwDBNn91y5bfzWs%vR$@iwY#*?3)h-?^VZSg7JX z>*0VqKoU#Bj}Je^4ER?TI2g9C4lsCUc^NvXxY@f;)z$1edSPqB8hnGgIhiu8h;#u~ z2caD%igZjS|n$EJ61*>)uBaaj~fkt8rIA@k6yQKaZsGQVr&Oudy%{-m2pMR9qGCmznK z{XIITu9~6Ib^24rPshujaZfZl$#})ga2{iU1AyjR66MHH0BqvrhQHtGpO*mz@nt`l zIT>{;Y*>cXtAVO~e2Ke1$9_`OX$HYRKNf#@g0L^8`e4ogEVuPc*$6l$R+hiCP8J8I z=Z4DgR?wG##m4^9j2?w*oPXz~c%dmhrK|3&2_sVp-vI6ME|8mCQCe9eer{HN`&)5( zZ5-yzJE?V^E`$`F30QjPHHzau=X+t~i?QKznyFMXq_+2I3q*|+S6_$+FfBFpJKPg8 zj742OTmEEY8<1yC?=UZ&dXJ7rkT+T3$~>W&z~)G!FAmVN|k8*a1ES?Ny9^0s+KNZZ!9(Kg?Cs zloxz%`^g)0!1;H3wl?dSfyfccBv-;eC#>WR)NrX*WUb{qQW!gO0E346#2=$8?-vaM z{M#IBrC~@Z9E1PdK^<8W+;E-4%J{?xe{)DN-blMQ;M}qcg`-+4`wBWNmRe_4u)|l5 z8r7nu{z=!^h;ft~i!Aliydq0csnSXng-syIxas?0j#7hIGe>3BeVu!*op1ia$HY9} zJNpPAfHA|f_u(nfnaBDPI3Nw2>GJD_0veCh_LMM~R6nf)g-iyyo?o6LMo8vo`lC`w zm7*RU<-&I><$@&fEm&7L+xl0}<4s3FVH*oc%{Z;}G9g}mbnHaUYPmkMD{@4-cpgs= z#o_kd4b(1RD-ZA`jk@uUd4xndII81;iWI3|5)pFUdkQE4?5BUSzUkj3b;UT+X!*5F+(*w&xX825Wn z=tff4liiUz5dW4CkSo>pM2gU@uHy2wC}s~j=1}pj9=rI;h~K&p%CG%^WiLkF_c|(x z%5P+4m0VaG`+|cAs4Wiq`g{1Mb@8#NfV@iYgwK+1X>=uu<;0Z+uR;)LIT{tx!mCIu zZDg)eZY(;rg=ff>U$)?fDkJ6Qc}wW%4j4NM(No~0qhUrszr78Y1U&dQ0xsoo!G;co zwBq}Ql9N+0bTyI3;(YsxNqv|24GQXo=yeJ=Da59vS!G^i`Qq#xoYbmJGStibuPH(K z)3HI&iTaw0Y?-jU;UVpglbTDJ49Q5d$+*`s@)2WVT#T{2M32+J=NTa1M#|5hpr`&I z84xs&c_;3T)oj)7*QqjK;)5?>z6?$Q{y`TH(qh{%nYoQzZ|FhCgIZ~fvTgfd*OSo- z4xXw_gfO7VzN=e3(nkzu#lX}lc?z*aW35cXb_fh*!$~D4`F&hF3|%BaDv^(*dV;D@ zAEOLFG=)(n^E>GtM5vy5;=HPX8U9(A9Q2E?P^Jh4#<+K~;T#smm5QKxWLU=303Ep} zJ4=2Hb6l`C_5Z=xU4>P-Hf-Al=`Km>?i?UUcXx+$cSs`*($Xc}(%m85-Q5k+DKP#4 z)A_vfTl-g7XM=5UbKU24?B|rp{xvp@rLK^_eiHXz=-R+}1}0{&UvU+@FAu!$8GvGd zqubxiul@jL>`5-3+)Zzy5a}g_*|6DIR!ZmKd*C7%!JW%neeY7S%j;J!&q*lk& z+<8s7Gz21=+~0+MCkU4`hN6#x>d_)vrg)lZmq`WH2-PDVsNf&y+RDmDhm~!aE{GsU z~QHMS{;M!A3jI%J!L1buPx)}q%5|?T}BvCL8M?1H*em0|IRMFN&+)mwN_1=$(~Gw3r|oODknm?}$K+TZ!rxMu;k+G-(&|VBeE&V61WN4!!b< zxig%4X#j-(Wyy+6DF-r`In+YKz221e=1MdM7kCp%1vNrA4q;l{hs#H0=3xve+q4f` z7bs}JyOEBpP7O(XBS+zlCAB);{1Fd1l1^}`1_0?tA+L9mQQxzNak-A{F=QiVV(>`2{kSWUQ&SIr7>`@DO6{0RCw=EXfW{mu;B4EtGKlk?j* zQT#3M=>8Y_CDs5qo5x`#XVJLs-CKAE)p_-IoWq#P+t|waEzxApGT32@1&XF(+1A-UDUD5$Rv=QY5hN|24|7r(Y52~e!X$;` z71uU2F4OQn>Paqx^N*N$coy8&t?QVtDy{?!@QnDQerWFjk<@NpL9QTY2@ba*P%To> zJ)Z(Dmc9)v2-%hcwe+^J7E_6lxKAcdJO(MFUY-Y*jMtRDX$?p*Ti7NDjw% zKSvt6VXK~hSk9MmlGG(9cjo9Ba`C|3<-)M-!2KI0qr(9#Xt*kmaSH; z1}E6|ub565RLT-8gD!rGVXD7AcYJyA$ah2@aK#Q@l zbFsD?%zb8rd26qC0(mT$@f~^NeMZo69E}Vq3N7T z{LjZT;h&+tI?d^XsNJApAWw!s7TC*@U3;uTf0B+_tePkcOpmlNUpMTNbW0WW55)$E zoiKtRj2-qSvsW6slo@G-hSm&)UgtcDm!OHUK9FQ=x-M(SYL+(3N$^vI#5q z6I>(3bxJQ>D>y%+guNmPpe=<8AwuSL)fH?Suz+eOu#QF~`D(%OQ**`$&>TV-ChxB( zv?7{j9aA}%)svs^sP2Ck%NQaPfKaQWIe~rr*}{vZLF=ZuzoI3+LejiBUq%>Q1gF=W zXAvJu#SMK3LHclPYguD6aDcO*rl#S@`6FB*p!I~?3JJq;}=tRnI@G?`oLcLya=dXA>;7?@Ll}HbpL8YrMssUP zEkU(<@Y+fmZ6y5-wWE?xlu*~~+$kHwJNR3-yDxB9@1A5TIZgb~yCH8wjm?E`iiY=0 zzj^LJeUfKEmlGT88?pvn?jOH1jocy=6ue;RM#Aj?DhTV=*e2VGPW^&eD_hKVVVr=; zig|CxD^I3HuAe`NBu=HN!nQ)jC~%M_=!;uv^EFI&Iaqrrkw*K`%SEp92JTi9IR^R) zpzlep77xP`=GJ+zil@fp6o`qeic#_@0JoM&JP|u?SaG+Hm@K<;jbZ>Q0h*keeU#Kb zU<-4pJ=i9=k!J5OJ!z^^gmyf;l8Q#1RB}s`zKantMvkm1?0GSG&Z%kR;A!2k=}DY( z3AWteNtn*Wh=*Yt9uelmG`r9b{CTE}||XqJnKh5^UC2x*wA6OB~gBu$s1`J31d6 zg34AxsSoHc zayZ-vhFf(XQZRJIKAuY}C)T`}O~r{4KP(`qry!)R^*lR{7$%@X18a}; z)u}i-|X&l+V>4&C%?$>x`B^B{Et8C)HXNeg@1VetwYqE zGw__k%l|6@l_Ji2v=e>Q#9I9pK8G|msgjU++GI}%mWo+@-WB*S8^SwafdVSmd4r-a zL?rjm7t;hlU^8Ow3k&2)?XK&|972l>2VK9)4-k;5c@HrrI%NcIRP|f~G zcAeFw{cXVQSfBHrTaVj5y7zg*+?6X!!_aM6Z~}xZ#fPUwoeK#K7v^%9ZJhmqNN2Bb zd8zwxhXtsAdP2!WLOFWSg8}k;>MgHuM3Fx|Y*zddHseSnii^eXms5#ry8M(vux`T2 z$X(;9{8I%oCc2kh)DdaVhp{nYYYLXf|6)ucN^KkIc;3l>dbvXj4>3PU|2er zaMz0?p}4Us-5*ou=D96n-3nJxw9-R28T5~PqJ*AGs_;NGml|#)fsE2v?2vOIoS6Wh zS76gn83C%g1div|`9-DtR{c;`wrcol{Ah2xvxkf)L0wQZI;N)rXq{Gg`K@5QY|#|- z1|V}n>^(ES)kaMTB-gOIoG7Xb{;G$7&a)ja7S4CXddohJ?t`NsRSOJ9+T>-BJTSFm z7U`yhBkJGxL(GP)Kl!qjdv-%@B!pU#(Mi(IhIyb4{WUWa0B%*eW4IvGNGEf@-?T;9 zV)B}q$>mCS1Y&%%XKUjK$UVr<+u#x;t=Ng~F^)3jEOYKg^#rAd9G>ejs*UXHh#-8O z$u{df@|=Zjm!~Kg;TwYn+r1}j*ag?H`zO^=Bq%HEDRo-N*fD^kuSD2=k9g8u-lRv5`xBj zueAt5mJU3nbU?_G^AK$m*9{Ifo}Ym>Kfijw^AA}K9iCL(oJPXDKT0D4Q}||GEXM{l zv#*mHDdF2=nJ3#%s!+#}E?@>jjv0WTTw3Z8k_4krstn&{oYmUa(C~+E)!xO9lx7%$ zCN&XnlbSds(4+=8mn111oK@`|FQlZ1O)CY`E{GDE&B1mU74Zfs8s-+EL?A6Vu)TaU=`-B%;3C6=bnzn{Q^(sTBgVxy zG^2@qJqymvgG7H~C@O`dqiO9gTypA2-NxW5+6Cht8t)2bIO=Zc{S1=TN(cyXcM>eR z6?g(LeU$OYqU$vA$CWgxO#Z2ZY>%mc-g8K=VkTsNmiDVUN*rjrKp7Ico}qP`uxR)d6sj!o5kWGJFx;X;k(4g<|o# z@#jL!HUm&OEC+ODpF<^ps~qF8hjT9t&bxJSqdf#e?yieRzS#- zWF;{fg^UFb7VBrm_K5wk)!oV7yh<4=HO1yvWXT zWdQwZCV>jBK9io4POzGx?t`XSw?T^HTYKpgCal;hLb52^VffZaiKKJtz~4kC?3W$4x1jCl>*?%nlS7u=Hx;lY zx&zHH0aXkn&98~gKWH7Ukj|PL@C0_VKLSs!V>(17lXaaOqkdZS8a8uBJ*sKrKn)id zXyD0q645e}zzPZCn#`lgiqXqe?9Q8l{gm0pC=~jD5`P#6gMl?>xEm$G#Rfu_n%~J_ z{2#J}OG%~zLY7dB^)8@3eAiS!4Ee*zn*{6xDx~jtMV1ILM<84A^`LF(KY@^?p}G_h zvSdyAUu3Bo7j%87SO}+CneAinoUqg_nVv?Vpoy<(C#j}c413-fsTTxEc6nS~P72evHHLB|> z2TLYB`|qZUc+Oi(cxpKQ@~PNJwc2hT>I>GI1h;&ZsthnKuY(%*w?U2E+n{FgZBT<5 zgn~5Kj05a?*SJHVQ`dcVpB@#Y`y1MP_!J$QIi%_Urc)av39>9`>>+{kBUt zL+~*{o7XmH8m=fwWB^|7(_D?E)L#wDiQ3{i3&R@u2`2RhuDH23p?+Tg@fj>xEFnYq ztzC^&vpI;~tcjCikY&63R-)pH?Wj4v@+-0UQbQJV=kc*sHpNswGLA!M7UYCNaTy9L zB0gf27W@C~w|%#eHlM!%=MqYHHPcLYBUF>|X=xcO`OhH+Qg@oe%VFHIrbTiN{l+o6 z`pXLqygEeJK)BTCU4a+-ireNxUV9 z#_Q~@C~6-JBGXHV_gp;9vr-!9kKoikha&h@1#7aNg^_*7_pXZ0+O8b{t`tZsuQDDZG=wq|= zZ{&zZ|A<`KTRqfbS~m%&fl)oybTD3i98m^YNZx9kRyM2Lvc_<7O9z(478{9WFxoAr zj)UofH3fP#%7*!eq?V;0zl6Kj@6`jGipyBi(faZvu{9m(2v?;!IuHWu0xf$~I~9N6 zsYEuU!KhAV&;8F?nt5}UF61@;b(Wl;ed_Gr{}!Cf??n+#Y6W_G-?#n-dU{s^|8{1U zII4bupry*&p{W;yH)zRAx~=FHTB6wW#Q{M}G<4gJY zTT#%Q2B2{Qn$y(Yz>em0>>Sc>LtJjsV99$tR^k07X-82lvZAyC2-8~_Nk-Y~I7YmnQCi4vHhoq9LQ6dZ|AUs={sS#x{tsG${VZtu+LOKrAS)(XOAMphZSf^FP;r~|cm-h(T2(q_c+^sb6lkQ#JdV@TVCMb>v-hM6 zu>+7};9Qh{|9j`Uap~ZC$N52&Pde&;JZ+a%!2zoV+}(q+omXm4{#X+guVek9Ss4Y- zvIV4COf}t73RgGx4F|5P9ed|8wxjREusf0~BymIMl0@b|8EozL<1&GLl)umpnN*ec z;9-r71wTA2H_YRxwx_bS%Rq{#LSOPl#K zTwrQvfqFG+uH!_kW+Qinu+7pU&SUpvmE7u4&0sa>Oxq!t74~s>TV1|he%$+UT#xVB z>$YAQ_ia)Gvv+N%@K7V(&5XVM^5wR@2G)2IhFnQr2d*MDr!aM0Mo5W9UWf0BfQ4d3 zi||<}Xu8$j29$oc)T~Q6%0qsMU}I9_XVf|t;tqkzL|9%bGsC8HWoz&o^;`n&Xnut#9@<1C9{an8<%oJpuwd|1Q(3=`CC7%ml9p7q+V&h#>O#;OY5ZH7k*x%k zW*|VNnd*am!+H`J)z{KY&TDCg9YzIIngP%(?05BmN;7Nup$f-zNmVbP(oEj^@y-90 zX2cy7+?@6?*D(p&n1_)5Q<@oid=S&VdMnM`(TkRN+ikv7f|K!$mw7T0(Sx0MrV)uyt@na7@U=V;b z`1117bz7EoTJK69MS_F-;MZaNxj2xRll#NxZefxcH9=9-^ z(pM@lSCj*{qopNcqip-JK(yD~@H$a=`TBc;sKK?BCS)y=+j~sv4_9=zccsDg&byW< z6+5C^&TWFDRdv=NYw0fRH8qnUJ+ZQe_%}79Xe*oY@=awnNcD5T_e>wZ@v`TR0W9^9 zm(O^>Kh{1z9fZI?Cl8ysz&k#^hAUvID~>4%gpK!P$@Cvspbd|q-kxx2dmL~Z2B@Wb z!LKrL+;_KC@~xGFcul0I1PZAmz*ikX7yp0a8pcW*OW>n z=xgHUZ*;W<)DWPk@pj9gPl+m|4keI)aPAm*ddB}RwiNV#*iu8FSzMw3ugq1R=Hrs0 z)9JJ=v3dE|b4yTcCNa4Y>)zuuV-4xsFLvpHW#J+E)8E`rWr{z%(ekVU+sn|JOcR8F zMSfPQe|I$$idLq%H%4=;>4(R*XjW>eTKw%P^u*bd;8 z8ydbX;5Kp4p}DYjXXd9gGhNn71ON!5@A`3Q1DpKiW&=T!rUa+*))SYwv$9>#FP?En zkpBkjLxm6MB+udeWbx`>ZAnq~|8_OfOIHB^Ga8Y57tpST;&oR;FWG;OxgaOqH=y>S zJolTt;0P2Ij_sfZtUp1@9vwvjh8+6 z^sza^ti%BCkUfxIIy&wSfK5r`^Bch9`pgw{kV|8)xeCr`4o-2kDWEYAJK|4N z>B?}`03nrE*H&`KYj1|SX@P<(+yd=03v4V$d!gikbxh!AeaY2R#8~PoRHIy=U4MV` zl*Kz+qmU0AGk2(7_xrwdn~YoEfFIq z{W_}IIY?hJmmlmg(_G6M96Lb{2v-T`h};^s5Himf;nY2A-%iVt85;cd*oF0uF0_3B z_gGh^Rff5>e9LR~QGtMYem4TE^p#u63iU=}HBdwb_mJn3G?uil`iEOGoOW^>@UIs@2-4 zOOVCY@FaFUJCp)d7hlxQsg2>kxFy$&J|T2?QGGNPMMb_@h#{_l8x>BhR6x@-W8o1< zW#$kdi+2Hs|C5$)A0b8j==sfi{OsWw`n~__uBN-%SS`qh-i3Ah=zJjmGhN7d2)(H%VpqR2I^j#OmhL zp!u#n*%jNdH;DHWXtMK9Yi2m?{%0t#q`l+r3CK0(E_fyMgijHf={opr8bB_UPf~d4 zE(oW>UL%tX-TEO<@>jLLt7@zIM`>aC=S|7pk^rgckgC8P>r~w;R7sJn9$(O>M~`b7 znOu=VuBk(nw2Mn#EI_-qHl2W42;Q7%KSI9Nov!%8vnP-*rp35VGYX6Kmdi-cPbIF( zD{W8t&0C87$6GQ0c}tL2Sy>=&2`Oj@4~`<1>{E5xXShM;jN@QYJ|p4vhjri!(LQ%| z%@rfVg1%Y*LsXhdytt;P+CaY(FJeJbr?87GS9V&VAY^=~t0*gWm^tpT(P9cUiIE4z z1kG=aPEUkEzkeZ7Ssl6hx$)3<+c%nwCx!FeEfJJyJNo z)MOUii8AB*KW+*3m0PNZDWcNa36PF>?&^re#ZDis|Yz(?WSvEuRB4W=RRU2oORtZ(gxaZDN=rkdN!wywgvQ;f{g6Y54Uq|<$ zu?un%mYap#32rsz86b1B2Pr;3#=oTYMyzmt?7<)+1)iw_{h@^!91sVQ{H(O+Jycnb zLj?UH;%zl0Z1e6RW7BZD?Hx-LD`i}i!u+Ufj814OBwGEcJ}ZCelO}-_P06O0d@dWI zxLw`QHYzxik`xoDy*oMIMSU&Jl>1J-mS%Llc0r{X;dufnP-&(FnL*^ns_AcOCeWd0 z6Z@?+70AbhwR>Y3w2;tl3%Q{@=+DnU)W%S#!nO<^&60X))!piBNKwFGh&+jlhBfx(H- z(IonY;!)>1h1;yrGVlog;Rt0^h-fs?PFZ0B46oXfmeNj(oFYyy9B5-!!~Z+8?98O=vzzw<-ym~o*{2r35Ee~m{a1CAV(*~ z-MhHfijw;MM7+$MN0*~%n#fAHHAfg3-%rr0Mlf!Y6aIa|z0g0ank`cFBuNXbSdE>n z6$*xJ&{|p{yeIVDYLcZ26IZ-$qD$`6+{ZxV8bD3; zf#=CE_Cq45G?V;Rnz`)VSNEg+w={!G`x{wW7eJq#__s8Z!lm5{Kc1o=4N9FLRMkn4 zHfoy`*b7#0377aAx9Q;Fp6ej*w!v=gDJ*~Lfs_g@7EqK_qSk7YRCrz0ptmm7I!S<5 zHE#D~7byWjCa7XjDs>uOVzblBVQUdf=E!(6~Tl}vyCI$Q*zWrlOfk zYb7W4{-S#N(0nQIg49tJ*xy6GL=>P^4Xwb?>#FAHZB>)hAYZD=mX3h1( zu2-XM!B1droe~kB5n5YGEjqyljKO+H4Fabz;iWiA67K7$9OVNmuHZ`h68|7bTav^o zWKeL#vCNbR;)ws=Y;Eh(D6X4z^`2}~CEhx3l%r1M#n5h!nor0n_&cucz5;pNb>h%94&M{7g71{(VH<{*=$9 zraS7hE?P@hv}?4O{3^vpbWu^>Vb{%wnEnV->D;4U@v=vvXQCDF{-4rJ;(tmr%ae`o z$+#&T5z`88Lg7!n^+^bs;@(O##eAtw9GL!=(Uh$NG;gIDs%5v`?()~tj6a(R9tFvn zBTgQ~KqB7)S9;%L0_WY}okdg+@eSGsuw`HVwvy+*4m zBjOZoT@K*JAWm5)CiWvLAsMBiE6%%dVId67U{FZ|4ac=)=hXzM#Lm zj(X^z(oF03YH)mGl1jM0r5UvlP-({68&sNcrAC4Ir!&%_^lHu^r_~TZl zeA#DZKH_Ng@q5+SynD@a;d5rjl={mk~NCNsM>s!J1ap!H64*uJP!PCvV3*YfhN{O)GPz=L3CAdlBGUG>KCwD;{@p zUK&Nhu^h&juh%`SQ-U@(ad|ZJ?PGaOz7)t=GBGvAH0C0HUR%53eF$h(QROBL)dPZ@ zrQLs=B~r3dzW+E&i4|#K#3!81H@=OJ>mDFy3DGJdxW^J&socs{;f<=xm=-<-jT`bCZQD%^~UE`L*bW6Q&Pu8kp8bv9)W1`ea49c zz7Z1Dp3dTf2~VJ-a#Md*ZRym>qEu}hmLh)^lQig3ir=*WdEu^K47;uPEUdD^h^&!) zks6-H`uVD=?f&d~CJEV{ux?{+ovtxQ6=Y*OaTP_xgY;3^TJRZUG-x&qkza8NAJ|8Q zqp_p{oO{8%9ci17%~kQ$Twpq2Q^PR>XG28eW)GvaVQe7@^tFdv+0oF*^v~<683tx1 z@8TPc>j#eGRq>sUSQr8Cy6e5Y%c0+zGH!r25eHt0yZmIoJg&E0I)g?+@X372 z{py!u__wA^{4S4$l2dItPTZ$dgkmkil%u~*nKDT76iwblijM zVMc5S<5EaaQzj&K>N%$c)Rf7f5F`oWB%6{KL)RdCt~S@K&3r$!D^tfg<~L2M^|OM( zArr!}L||lhA%;auQHig~L*hvv^#C1l)sibgp&?(}H<1ieR0E5Pd0@Fn7i{)u6@I8! zymz9k9}EjJpDVoAcCc+@RWNmY*uTU+B_7=96IS@p(9tL}BAY!@UxFXwr3cy-?BAx$ z2h8yI>{ra{#|=ZjGI`8+q*jv;V27iviqTULV*0y|~Rcn`*}$#Vrj$HPWw(Cf~LdKS}hR8NGI26FM+uGBigz%%576$EV z&tKux@7_Y!Hu+jW-G3O8Ep|wsMfufP$|4V$)+`ze+*spk3C3>aYVq%spPul{^xzNq zBfJ!tpT&`iiJHKmomN1?xby8g^>dZi9i2#9wj6>7AyS1oIxHwD(~{za^Nma1x>GKa zx{Z1>NFS7x5iy!j*TM*@x(+a#DaD0q62)DMtW)(L7)s{C45U(d*?Gs%hA58Jz#{|Q z71Ic0rijxQ6q8DxVx~84*73p}$7~tS4VN@Xkt@ylQlp8qb?GH=1%Akdd|)R%x|&nUO_kTGLXU@BqXKS>$-Fymc~qqn4t z1Q8L0p!awt=2M5s<-R-ymKpuNcTCwfyzUlom4BgR-)GhH_Jjqv-R4!VlJNoN=!*nY zR-1oDKCBA+Wb~}r&z7I)AJ@;zJxjn<2tBUW$FLr>;{y-bsrHjo)#wXawu|iQf)p#Q z@q+Uw-#=wXF)O6| z1wdO-lbH&@yDZZAVZbW%nv@Z^#nh|3NQUb5Z9JQDg7dxC(xiay`FeF6(PxXP&7dD& z_c4-J%x>G^(`caAZ|~*1KYw(`%T#rlX@ehpLg9DkFt#H!1fPp%3nCuCD^(1>1x22; zlux6iNNXu{%OG8dw3N}^&_tjkH{VWeBv2~{}BrCy@*Tp9J)lX zN7LO`?3DR6d||TDmxD3&DNE-Emp#NdrK8PFJdZu(OtMgS2h9(IZj&Z{6W#SsD9b?R?bAMF=V!03=Ej3nLho-$Pd{H2J8NDI72OP`p@u?4?Nn!cXiEdx*1@T(X5mEr@soRgs#vDvp|P7nw;v@bR4uRN?FGp+*j+grD5!q zT+Mp1^v3#N%;1fZ79~N!=2^+rD8T~+S|oy<J5wr@^7(@sq8%Y#@0WM1`GB*NZIS-6b!|N59zi?Axnz&kE(uvaqK(OxcPtH8se$@ zE!8$rS{QZ90k-G;DALVrmge)g_AZ6V^5B%glP*Ttb!`x~Lv#nzBGJ zY&UAL?5Z5?GwjW{%0LmhjBOLuIu@n+xN0&^3$f^mbQUlxreU;C8`8A8mZJR8RUUKS zaJn>EJGe@}m&!IbMY^8hYuWT{1?D&hc%O805w3lMw|jB+b-t-r69a+}ybI)HQhoYr zdo-eSojw0g8VZYzp7gLeLlbw-cZ<;eb{|Y7p>}B(L`PT}&CzyD$r{{Uo9BjI7Y&C5 zXFb&L7wh@1X4d&KGcm`swD06FH2vc^no}+D8cmKn=WJVa9L*+559d$fH7uG#@Q#sI z4KvD`7)H}9tz^ssSuCbR6Ia5-?_x*K-Wf1t<2k6>&lEtbZZqr)s(vZDu`@(CIa%jh z=f5E=RePuRq&zoe7xr3|ab*DNF<=FGMmt&@c27`%q$MK0ia)@NJEC=;^>LU3#brvY zd$}j5IbF>qpU&5wOuXGsY|JEQ#$0lI)ZAaCY$i}h7J@LyB=;-sCUHb0=52nAr$Zj4 zk<1^-L(v_Y2ip%&;F6%jKWEg%))QuHUMKOSn|KNE@$_y<%JZvr^Y+|j!#j@}M^c)9 z>AgfF5@0_4R>(F+`=6c+cxs^Z-=0hyK=HLF(}sEcMKwcpUSkF3&L8u4|65PyNA>^P zlW~d8 z|Le(Q(+%XIf_gHlW@Ag7+WNa*r~q`OVRIoYre(c2W=kF8bkEL}EK3@vCe28x9Upr5 zJ=}o%Tl(+iImw@sHw78pH>m zEO-%ww(q!gU5^BXZCrQLq(9BC8B3_myQQ2jTdvy8^r~+?Wr;;W9y#{w1M;fk`DI$4 zIJcoT_Gr8jSelO{zl#?rvJ&%xj3Oz1l!4Bx{qYR$zzD82mJfK0W`HM}(0wxtczI3c zost#TkxhD-s{v)N&DyHxc+bt4LK)LY@8Wg^i?Uwq^QpKX{(*`aHY+U|x$G}oC%T7` zUi}*8Gh__Y+S{Jw3D8?lPc|vL>m-?}`G@d)}K&{v)5e3wNC7@20re)y~otrmTi6V7jq=R$kLv6h10UA4YO3JEdo~ zZq{4DghTkuPMi|=poU?360(IJ6F(z3?;XPoCm2cQ$i%>zczd`g4}2Id%p6~Hq21OlYEygqm8OeG>vL<4 zvuC)HIkx7CS{x2dXpt#b84Qi66jbl9w&tRPT(I9IR8J{7y%<`%UfRcZjGW1QX zuT2^EF{HPq%=q-rl((jg#Xn6MH^Fe+yU8>Ms&d~-I5hLuL+Xlu&f$=M~^rPgz)DmPY^eeSw zH@4>U__A~QL;ftv+v_TS5tZEzy1l$y_Tt9z>a$>wjG)Ja<0JmqR;TZ~cqX5=%g` z%$_6UgmaX5T3FK&$+45?=QueuN4uBLm)oR|XR@hCtkQ*ID2`6kItI+3`w)xk>v{|A zm|s56B9l~P6M)UHFK(8*{%p{LcJ{n-@tTJ2`_K~3$*&c5V=69Pk^c|1RGh(GvY_}e zW1f~0zI>Uw`7@SPo8g0F%fp=^aX62Pfj|j_I{GYuL7aP=b=Iur+|dL9kqx@%r|D$O z0X$WJpN@2RkXYU&7YhH&3{@b5YAOpLsHUovJV>dSRbHxpVV zTFXBcan&cjR%LWAXv>is9pO@!e{`y#Y`ky6J1;_jcQ8e^`kOZ_=NL`0wX0!+_cT zx*yxI^>}ycz%M9Cbs5a>2_RdwvX;KoZp8ivTLQdcOUO%c&4ZUXLy>`)i3R?kul0zo zNtIo}0#8h&AJzq5w3|pli;=VQN)WcRxCzpU_%mT*fGYQTWI1+D^qvqg9@1_c7H~7& z0QuDX3Ig_dzEf!by(O56z7aID&`>MCzg-!fn!r}xypgHj?cM8mb%)r3&0+`3crjLL zG}qtp8r5$4)_ad5%r(*LOgXgpO&O+B+J9qJndbkz*+w{eRtsU&D0tJBuxh7u4xMA0 zSp)uROEFe8=mu15PnxMe|B%Xu10=S=-F+gmg_7wGRAd+C612JYhlXuGKCH>T#FdJ> zXi>s4#sMF{-P{q4qh#C=f;u_(E2q!m>`0%la3#oO(^c{5Eyhu%((+L^m&>kj6BHbw zTx!S(w$=5y8F0z8ALKP#;+}4oFE1jf~4U&0}vB^%8UvV32IX7b4+$0{CK7j2%R&dbE#O^(wccPFebCI&@it6q5Sbh8A` zq!=3^|CkaXLvZGj=8)S|dzY6tm};cC-sKp-qTO$cW3<`GF<4!Q-gVF==7n26?soPI z;z^3dpKC4MP!?K85d^VB@{Yv_oQoE>f>~OLqIaz&188)CNBw9_8u>~)@D~qs;kcAw zvRuNdk{Z!FD%*ud(ZfSs{oDJU{8ocLzpyMic{G<-5iaea=r+(FD%QEpTYl@>-fT9fBOm%jG(s(HV-)3T{;crh7no9h znzV^~YF14d7sY622~nx0n*~J4$?u$YMJ7%r4_F5|PMwnSGG+a4l<{s8eUV2hm%?xe z#oPXgZ&Tl49JV9hvIF-A-$7=PzQ0tUcGPyBRrd#wibk z%I!#!7n&$8WvVD@9RLis0rRKDs&ifo&H4We%dA8J&3Keuf2}_FoMnMVFW)@Bzktr? zyg*BsC_SHHm0wy9O+bj8rHcI_Iz666kc4-{1n!EHRa~Y8z#;4}7Tc zcp2CA0Uh%`t@oROFzpk8I7NMM8scK0Nzy^UWp5VI0s1km3e(d5yR} z&F^Q9g#*mSu(*P&X?805_sJt1f8S#(P6h!<@1yu1#n*y>#Y~jgpD&1Rs5+#YcLm7s zV54Bb7ssN;E7;8Xi6*7Z`k4y0)-4D>b8-WBLDx~Hc%;nI(ptT{3HXPnJ8H7*pQw#% z!$+^K&!T*K0|3UWF#IcYUTT~`GrK5V;C)|aa6`J$#lt%H0snGI2JmHbZFBqChbgm6 z%)zdoONm`@_F0Te?MhwGN1->G-?kegNU*^Z4Kr>LcIKn1um<~){;Aa%yqu|K9V7cp z?p%HXOMD<@wq~AIEl69E!w{nTt1UtHD#`K9VArE+NM4JMY1|oZnfPxyW9}c%eLM1P zVl=DOmtD}W1DCzFiiQh!gq`plk|dS(4;RhR!#j$_ii0NfaungynE7#v5E_ZS5m|nP!)kwoum;dppTIr(LlyR)Rc6UVyeoa*w#NKEH<9(rWnSg zqD;r^3!wAj8Xk1nVbe==rs_7mOfu+qSo;u2hn%y+%@t?%iC&X%R>qD#s*F4xrB5)9 zVk6w$l|)@PQwN;^^7Y{-Mc4wamotksNgx$ zW@5(+sIm7Y&CY+IIh!y z&u2h~b~0{m(9KsQ7niiN0ZCE!_&T#7FP{0oU72`{A$L3xffP4&a;NN-hT^iXux z0`+(4TTz&ZB&LwqA>AlgW%I5_3f~8L(k)I#r!~tHEI%aBmf`$D`-2q5b{U*KK|QM= zTn)MgN6Jh~nLK4Iw(Tb8pFAl7`~R_bS5I}c4Zr7cclY4B(BKf<-QC^YCAho0ySoJo z?(XgoBv>Fg%;LV^clMs6nK{@~d#awJ^$&Ed?&|(s-w%RU0->O|Vt{FvaVf^*mq7!x z7zyek-fk0E@?%4@QY1*mJVLwY=1Z~-0fcF$^X-5`I(L>Y?by*q@dZ4BTKo^U6 zVV6c1{QeUUYtAX5aLE(9ffQMQKzh9)9@{KjBozlU@D~*;QCgY)n0`6281bFOM*$e@ zm4~g*GC*jBI|wUK?o8%_I=O9Fw;e|{$c0Br42QXoQ^@v{S^`~FiFY1_!y8JenbX^F znz=u=Xlr61(C34cTm6?>0z{oGL7hO#3T1{7)I}I`eVqktTa`RQ7-5bihSv*MHnd|G zkmJ%pOeAw?>Gr3bqxk2)RW(2A>Jje+X zsuG-V7M|hU+sHvWk9jsx^DYb_g=F0E#KoO)9itt@9yYV*+GG<55$_o!eI!T8n9LJ6 z>Nju=qr)L}qFHcA=jb!fu+Avj!TQ`)Y(Ug6%kPti@Ku;3UNZ+cCbKc z$(kLL1{FWKgoJUX8H>;4e@IKJ|5H^47mU*Fvns>Q06tb+Pu$)<>(!6)S(T}TS!|@2 zXtY-mL&S4(qWbnnf zLY~XB2sNpnPVUQ^xaUd(FLwVN0*>HxVtl}ex>u*b527SPX^SP4WOX$Zys|M!Y8Qv5 z0D~5~;u?QgQ0Dzm-bhv$!i4NY`rya3tdbazkwm3A6w>3f%+w@&me=1=gPP!+edTMW+V}!4AZ)ap;Chhz4)b|DCv;k5k(+w__FI9 zCXcskeG|{zCJZ7CScK@>363i8^cTg(zQVtlq`%Td^^ElRB+Q)dviWPC6EFrb>ic}h zEO)BoBUiBPk&iFfzpOtL9YC{+5I{K;_!@kS> z+Dt34JW-MWI&-A39Yt)1)KViXlb>7;-0F~r6fR{!h^C97k@Z!{@`N33s3+`t(uQED z+hF6A=s<4X!=Kct(_;Jkr2|BliBNB(lN(rGxi_Ah3ILs}5 znQuj6enSzD$;IHB6U?~!3XQ62?~7{sCQpTjSb14O??Kl-FQ=J}=7*hI10IE+!0WOr&?vV7&4*`|7+2_d-bJUWT-?GX;TvlnL{V^86vP^-VU1t}y!25m z#1%yrdI}V_MMrb~%u46{ppjk6nQPq6tx}Jw&{uj>C^+JNYN3O%oNn7%5%YVBI1Y7J3!win4Ha)3DFGUiUUnpzh#1yt}As#SAH5vWP@ad*jL8e}7kn?UAHwrag4R5eZ&+3HM$(v=PEwNOg-Byj-V2Sw88@u4JH?=#5nJO|lbBl0cRW*G&=i z6akHHEzj2CYZ`nk!#jN%-S^<4m|eAhr6qHdf25_!sZVJMZ-|#Sg55Ne{RV*l}i=TZW+P^x&3WUgK`!utMwzC^L+A-B@mQ8vohL-A0pZIR5Ma0pieD zODHJ)4#1@7>RQURvEOgUiAGK#aEZ(?zyn1B=k)XoHU3XQEU+vT>jZmt$QAn#3s>i{ zV-9L0zpz^ekUi-~h!^h20smu?3t~jL)IWrIoYP#!ri0eFlidO7D^B%Dc97iHBu*eg z#8O@?hom2ID6j92ngkzyi8_bTf84PZZ#%pZ1bM2CeOUP?wm%4n{#sx8J-@sKjXZw5 zJPK5zX8fKeAvC_Z$R%%@cEfht2+Bbyym`pppkr-ko5I$1F9=H_)(18TW6Drkf+<)h#`(R! z@9aF7qRB@b z`Bij;P=W(l#%swkqcyViChDd@lrn~?J@07cQKs~YrgxBs3qw_0OV1sUD=x3pp8 zmo2YcSw{TKmi^0jzn$}rmT`EoA*K&T#ez*Qqij^Va{ZR|WM+PlRN9OB`UKw<*Jl!E zNqDRD3X)ulHrEoq5XFAM6+OJJc@wKYlC1|)$b0y?4^+QBz9v91eMfVa*ZdC3(0|TL zn-1&2)q}99;Uc63ooxLT+HQAEq_!qE5jxf%Pub#;!}P07d$no^lS`SI9YRq{jL=zp z-kDLh<*k|;0~(?!22QFNi#dsYPx_^*^*d70_p*645=EB($}_bkeD(eoX6vu~fN z3+&@40MbgTG%V{CY?jGjK~N#KuEkO^Y|}yC`_{OHmp+@P? zBsG7U$3vI&4NSEL4*}|zJ6(Up_;;`Q-OGae2fl*17-jFLKyTKG<#P0u<@$aUpZP9} z{H^|l&VI9rv-*dC8Y`|<)%d0IDQ|0?rE%BwI%I3d>C*ULpq=Fohc5#9(rog`QaI}l z?Zvyl=p8%I9K01zC^tx-HbItAzs)%Iv*Iet5SN z6r8SEei@oyhUnR9-Q?cy*Bhp-0W=|YPTzMr{j*i4MBnQ*-t%GP?Bp3ElZBJRsObirz2lH?1m;aX~E@d+Ld9TiYfEW4gKeF z7Xzexy|hk7+44WM(&F4U)Xz=%Jm6R0=s)s#o6}MQ7^a~5Y3;WU@`~+vC}L*1?ye@R z%rB9%0Q64dry4HX7dyCH`?Z;p&XQusm%^!pm9++`-p&h6Q{tD}J>U9u80!liNanT; z9_O>;%Zhy+n@mjS2_~%S$3a_G=dsq3I@cf1E&FSfKRqO|*}yNC8RglMdlsV^+e`%| z`eKQn=UbaLuoDd{ur8_CO2`qj+m+|EssYI2#2W(FwgZ5Qd>zQna4slp`oK+_L-s4V z6|W}-Z5gg|t(i!&jb-!P=Z%v;P5qkI=i8Q~(hl=t>dLaN?^rk3?#D zm$eDVdD}R7DccTOlG=^(8`pZ4ReT1Gyx1Q8P=}Y?BQnvm^27OY`1bH;;KcP~r?2ne zUN37bz z9{E%XYBQ?mAYy7Y`1ZkJO5d(shBZs5+WD@#XYkZ!egFiL93nzau#V9*JQFEt$B#~$ z+Kh5)$D7XZqF#3O>igeTk9q4&os`{J4|A#>>X{8^{#**g`o$(PNih&5(=MdvG&H1Z z&aDTkat5=`EUq*C!R5V3TnqzOs{rM7wKfh!A@dd{Bt#60v39Cu096COsN1Fj zs667Vc6-!fB6WexK}lRPgp72S@L8(3t{XGj}iU5cFjx7k4LHOFN>oI0=kzt(V?uVw_y=! zU(CBeHhG8aNyOB@o!*SHN zpOlwauIwWNBG1TwU?nF;-rMA{>N)u(Y}%TZM=(9F#%t)O6fZN`s4<#khOjNV?u08G z_!zHZQ!i_uxZ-(%?7Uw3Bq8T}iJP#8D|z})SPAp*p`cc&6P~`-;fL%mzX>NW`iYN) z^9QTvoj*XOYv<9UYHEHZHq$!K4P4OqcaXaAad^|xVl#PKa7fp=}~1GfVfKJ^PsT zyC`B^vOC+DQMM|KXK_TGDzAg9eOIw&K!ny;{F*OKsi!jf?ok>|^NY_nkoVu|M6dzM zeD4Y}+VJa^wRhZ-x*3A%=|8%^4Ni}?Xhzg0;l2Z=7JU=SMDdDm{JZ#v6-XvAT`DPU z&c|!AosUf0vV5+{KO~XJUs~Y;Ck}hx4d32I_L{c;lY#l~42~^QG>Ft0242(cpIOfU{ja!@C5L-jZohOJ~3NX0(>to}lq1Gi+Kpqrr3bAP2HDkeGSd)(98YkGd-_jbN}9^rUr)$xuUIss0OK5T#$a__L0xQ7qm2^hSg zZo(6ZB?#3A{3DGiecv}hOyT!-M0pITtxHiA88b449Th~wrShPbIxafS4AZ3_iwbdC zX==j?Xqmxg308BZiauPjFg3|ABj;Q0+33mQTu%!6t3n18bJ$$QTTu~du%Roe*>&1q zD#4F`WX}b%83rvuvCh>AQUbbPA^@-qSP|bX%cZqs@I0=^P@fDG&#NZ{es^S@ z9_P}6tyetBkKW)$a27Djr))I=P^bGSrOF*YPwPQt7Y+MB}9R`5-Np?XczC z8bCce-9-%tx%T=9K(&!D7`D3g0yZ{n%)wY!0{HPM9+ z2U<4OpYeDrM90xE4k@8A_=;0o`|zZZ^t;vmN9EPkw-3KqXT8}j6n=5ZgA?7`JyygB zCCkwYA3X&y)hPgaDd-iHC0c#;t}f=)($xH{vY&_2%~}3Ds}X>NE2~r;AcBTU&AiLW z{q2VEBpjV~v0PKc6mb|H3ous~Vu?QaVVE$p zI_v)a(E9;Qzg%2&NyD?Rt^xNjRG3GK%ek1hH#*2cr>58Oy2^6YCzdFnpSaRw2&vZa zC$1zlnyv@y63L^+6r7H#LyW|)(X6*qA^eTPB$;uw3UQN+*kbOWg*+?WF{SR*lX5~g3GZV-f`kf}GZ8A`i;C_$2}qy^EpcZn47 z>0uce+!$RxSTZOb2bxH#E`|tQECC#S7=g9}rrq?Qa^ze{?$*+)6xA@Vv5ZV5jub;w zaty$ut<3NYEn6VO8x*+x1jVpnR)^@N9Wk7#k6w3p?O8%l(b}2~GB^~M8vt2|Cs1n* zkqOaAwG-XJNL~;}faQpgUOEYyrjZIvVcx8VPVRRKzk)!VfZ3wAFgWtAgO(f3CK%mw z1T7fuimwet3fp+@dqygV^S`Na4F@3(U|JH&T&W-olm(|r=qxbQTgAL>C%&y@6>Uxb zaPWD;;51wr?2*CDs#dz=ofZdY;zBj-j5 zMo`g=T3X4E&N;%D<8e9tv{NY&73KFg9v`kj3O1`;y8Jt2gNOudeNO@V(OK6pv?czr zN?~}#H!Zt0tMb+nD?~#A$HyVfEq}gw!lBc0o{~Ua7a($gK!gPw-pLyEYT2$DL)(1H z2)~3WjHJFnH>9epU*M1`T}sJNs^m^0F7yrV{s#pNXLR+NO0kHILTYFi?}FB2JSsta zD2!LZYh1G`g=BP=z&K<xqpL{*HzD^A5!)VvP2z)+HH1Ll=n}r8 z!x%9cxH?W>lP^cQBeyy*;$ICQxTJ-ki1bg7!A`woLg@=sg%$KP1ue2EvGZs9rl10s zYU5kMC?N*J@o%Z;iCsjT65CMr_3cvG!3d2CGluc#oz7Hp#%)D5XfV}R&7}=cjEUZ z%%K_2&*M8#hO}H{Y($3@>6`15V($?P1^`plfM=qYThX4PpcN3a8s{#A9-o^h)%@}9 z=XimZCqBW@$j*U8Y((jD3ZjIOb~{^FvuU-4erHM?a@>(SAshNcm`k+`8l73|>DfwR zO~y1aDBd(-PsYG@uEka)<^ZKaS_PFvj7T+mGz>#L$!Lf=_TL1F_WGiMClwe?mJ}oSOQ=IEs2$bQuUx`R^#&y$}Wfqw^(E8=e+mqA$}Z!=^q3t zhV%lj4!=77zyUudOI`mq2r_{=m0ob1DkLhH6a72ugBA@O380xo1-C#^FaT0I(Yn%X zrJ55bh|-qH#bt$_{`Ei1a0 zUK9px3vbq<&J=Scy|htc$PH_{AliqQ}93yGkn`2@;BYx z#!LZa5N+v-h?%Jpqw$;x^{z*(T=am`^i0hLw#*39(s5fgD z*YTg_DuL}5XtkA_v)piyUSqM9&XW~K(8)8%!$iSllBeFGNGRnER~&>Nz5uH8s^S`Z z3TzElEa`eLPAZt zNy!c(RTOso4?oi~i}@bG><@-f2djsF4uP-ZB}iM26~>dAX@O}OK%yBolC(KJv+(L3L1Ss{c_P}j-|7gGliT(no%th9oAaTJ0{ zUTPb+6Dc`(p1DOeN`Hlqxmzh+7L|)vhE-sYs9UaBZxmT7uDo|12vqFY)4~njRMdiQ zS!IhsgUQ>-ofrgljA`478Vt$YCwDcBGgI+to1X(h+Pg1A9W>Dy`j+Oh8!`-3f>`%N z<{% zgUq+@4z$drSgG3CIwTV0S-@vL|3%V2PV$Ih#7MEyt+QvrxKo2%R9cX9 zf)H)zFiQn_^@aJ`Zf$b{Vs5~&%(Bl^)9mUPH5<<H46uEmA3mI_+6mS@EA{Obm19a^3;2>{d9AnRR};%1vdaQ zTuvIoe)5AlcnCA^c>8cf4c@{wsBU5zJxDZ9`IuQ4=&}CT@$>Sy-u>`pL=MJFn7iZ^ z0@Ir1%%QGop#Civ(1j!c9~iNS1?rqdhq-EeK5|lgbw&8xEf@DH@cjN=|FuI^o9zf3 z%<;au&Pg(ICXgs_+bJI*`D{XSj~}JAC6@;Wv9_lWL(!edO?(Km5ThP=aQSKN&&zvK zY<1&F$nc$+Kc339D-5ANw0K|d{DxpYU@hi;RE#FO}vYn~Yf5_Q%?oL;M_rEy689`*dqz3o-;jc=WWS!@8++snh_vDokNLeuYRt*KiNC7Pzh zrB}R{9|kEecJ1M<*0SoYbjNdn|GF#El```{3Qv2G#mjp5E9N&PJu=yLE1`I&F;wmp zMF4d160P`ehNb)U2fvr=b-fq-ErG7T^q0i`4{zJT`D5PQQ36r_;7Ti$gmXJSx#X*c z^(N|BPS`OMF$!@`e0gJ4?QA)L82^PU`IiGUcygHZU2Z)|T@@{nD23Sp#z0(2c^d7c zeF5HFVB62Xb#f6&LfQex`z>`vt)eSGiiM?-mAtSS79a;r9Fe3=q`_5ZsL~;Sk@O z6&6iaR^bA#aHmeAUxbm4&3@QpiO%Ut3OWZyZAOqp^by!Pudl-n%w-+A47S))i2%zo zd{F-9Iq4f#5?Ps>>><)RW)AQ%zP?4;90J?eGYiSZc5hnjby8^h^R|))E6Lc$Q3>;2 zwEnwIZv*BaH^Ay7x@A17q*L8A^;(j?{7$k+y_J>Z5}v9u`~;LLN@?_hKig^!-VG}R zf(_l#QFwwTUSWbIYwhYf8R%Cf)l*1oZ&Ta_1Cf7ZrS{LTO!360tVA>tPY?N!a2UQ$ zXZvN?^1JLGly*q6dN+vOI(ZRS+nPuNQ`gz3{B{k)Us{0mV!5nMF9%W!aP5Je6b+GP ze?j{rZZc*0Jmvj(%cL|KXAI|YTx_@i1MSW{I)e{+!lrhD%*#SfqT*+zW+1GjEtGEB zq>26bJ6hFp^gTW5s#QrfUXB>rpcJqnnUq-QDx0-9CsbXzlv=UKt zW-4ltNT4yXF|G%)cBVZYA0@svYbyqlic{h_v1TXP{qu@=3Y&6|mVk2DKxV^8F+;Q4 zD%23a7F5Ub7ik>!T&zYW(`bFx+`HHsspt~tq}{ukQZzCbvxa{~^e6L4t4|3(7KYvptjHt(z8E}fU5)cARLMm@~ znk8=m{r4$|NhZqrOWd$=-&P{w?cRxMzl_}dl&y+bsLnQ?TgA79qxSgISxV9fyw&Q? z9xG*%j<%UlL3n-ik*yPmvqf-OFzM&Wkrge$h90z1R_7;0BK7&GN0^Sjs8u+@Ljh~_ zA^G61_qVC$3GI#z_2}Y7*~d}S7Sq^dvj7PUfo}^BcOjy>pIw=+WOg|Rcqa~HYvQe< z-L_`MK`GFl;~ltuuJqNls<0x>Sy3Tc_ThQQX8pxb>&y)3gA#gBQ5}&-AmX4Dg4xr; zEmRQq(`#%V*T)xd{&~fQC%nI30n^Ad=&8}`7JB}wtpLZw(A9*z1#7pV+pRi-`%rhS z_LI5mNXh`>@?UFH#l{4~ycJ*FDm6|;VHqLjLN$oJAbee|3qhoLcn_vB6?e6hcel>6 zj77|Bj+0Sy+i$7RlkZr90dk%e2w?}ieTf`QfXOkzv`j~_Z+(b$ojM{_1xi>y>$ zhKn{WV2&O9`H%j0yvUdKbhW=}i_!FU=nQ+Vj|^!PP`0;vuY7m@z-UH!j2W>V z!q>zOj`7ILFkyEEuj37~(X2v|PmqgNmS~(g6ZZAnNPHne5PZ+bTYA<9o{(EW{rhc5lOs5H`aRe z4I5)&fX{oSXOW?pv5RIW0}g0*MQmZtir%xt@oDn)(A@q_2`PnE&7?rNp1%n>>6 z77b=aF<20osP0vGXS!4}^nGv)-Q=`_8s9Uk-z(wOuL2HqU0nl8kl>U=Ilpa9?=>^&jOG{rxRu%`Wm0+FYJj7Qz(@=(jX#=>*q z8=abk)Z~&mh-g&6eYgM4!$}O{CF}Wqd zK9!zBNJnep%#?o225(C>c$sQBzbOAMxepJ#YO>F{*;|Ps`X*^9rc`t`{d-Tc3&s&sd#3e{z;f7ywb@sKP(H#K|pElpVI{p8oS38}Aq(f#-F^`z&m%U`0;M){~%Ck8JT zeX7_uc(aV2I3`cV*OWFre9LS{vnJD(uh^ZBK`Qguc#)Tb>iZ_Sz@3{qnzCxAbET$- z%rqOyL6yJK@>!}zwv4*TZml1Wk4Tl(f;Okw>`nQ%jfa(il_0a7AGONB(2SyL=r2!Q z{1rh1s1~xR$c~o;AYO5{rD`RHp^UNv9x0jVSg1QKu$jdV3Qf~DLbR5enpC|XYtN!V zE1#Pv4}ZL`b-o|pyjtA=Qf#*>3-At#d49*>L)oeN)w@(>Fm|w6wd}2&sTV4{tXToD zZ7`#31}rOvRW|auCZZ)7b;-wK4px7dL~R&oz?F+>jdPyW`qO$;S;q4aRDQTKL*5_u z_H=&~ofvduJnqY9NcWCQ`2~@8X8rn*Dy8lMM@>`X39Y^U;PdcOSpFcqLn6BE-I@SD zSDc_s{U!HGF|U`g?icX~NWH4MqCB*}t3DLb(i=bW#UllAAwonTGelu^SLncT{%o8mPovt!X$wX_yv6Ok-^Yhj-D zL5kINx+f8EXY(($J4hfNjNDcrijC|qwR^rJPnqplg0PKTv@KI{sWjxNZhOKL6WPQx zQ3=;NfmP3Fx=E)`aQdEPNM4LF7Yw*+?cM9AR!oqchV%~HR5^BPq|F%p(4k8=N4VH% z;{6e6#NIH)NN=rSYelo&&F8s?#qIev-!*yy49ui7>EO{$8?zH@m+k*nY+R)k7cBSb zJ#s7~z>{jC+t5+$I7}ju)E)Y|x8SwJUllYH{bXp#g*G}t7&IF{ocK{ca`ZbJn~7cf z5zEFUMOr>=vd~1pYdytWj2~%qudnYtKzua)0M}W3*GX(ZKheW6vvhYnkW$}ln>n-z zVwpty*Pe^{ADWXw;nwtMyzUac^7`d{NDttu#{5aD1IR0RfmKWx{*7k8B{VKbiCYYA z6Z0d-gxbHjh=GNhN=e$EIXR(XOGRz*sG3W{w_HkD@X{Rk*g1Z0vy;hJz47yU(>r~C zDDLh3aQNtqJK5O$ut;_6)8<&Lgd)QNxsGq{y0MK&doAb1zo>Bt3$ut>{i@=9XH zU;gEl=q+iB<=0|Dr~iI0=W?cQrRUDWDGpb%5u9$5v;sPCa#G_wPh>dwN@?dCEvO z2QB;xmL_H}xm_;}s8URmJaL3+(x(v$jbz`^K0K0}BX+Ts5)ar!jPk}OKN zE5Wlb;h|^xP4&eM1&!)kgCh`E8rodud=m;?=(1P>1;kj%*8NtXGQB&=b~UD;SjS=NguYu5n&*7Eb{msB!SH#*5HyiyE^!I>Y0Wq znDz$;=ey3^a^82SK9w!6G@FJ(t&_wh^`S0?Dl6z_vPdozyk}jnp+q|GDl*LEm(9X2 zyP`L=^5$;w@PRia#pNxxL`BKhfmqI$MVtp}Dxi+&#re^z{)?ze8F$WHYw-e@Ta`k3 zW~k9(h3o<)O&$rrxQxc?0oZ<~yCrGu{j^~QtmHVyR>Y)pQ)GNH~WdSlj7WJq1^!R~0JQ2_91@bToYz z`PQu^xKZlk3LST7dg0n3AlcAT86wN&E?(8nw=YFuA#V>Q>x+NCg9l!?n91^r>{thX zUJ3Oj*Dv5i>{*{Y!dmO2w)tbr@Agnq^TYeP`62(KJMP5nS^ouLfZD_%OaIu_>&tO3 zCe>e6ZjlxF_ZjryBw^p2i>%tgrh^a>4mZPaelXdi}tU2>@Aa@ zc>4H@{N=It#j^+Yqiphn7R%RNyz%O%l0D=UC1R)-&aq_DT};h!W>z&*hIhoyqX!!M zt)>{7JEpWs$$Yn_jcVLvZSC|GYTH6x`zj|!@**v)#dxNcy5(wXC$-dZesr%5PDi$! zYOIdI_^}Nwdj8k8Cc9TanIsz1r4Y}ia@?BcWPQsH>uEduBPm&YS#u1gcK_p9e&YQL zG_*YMxnXC*ndbwU13`BQu^6qiHkFXo`k%s#!QKY_0xqTkuyE8wN^qb~%!7zZhJPA2 zwk65utA#iNcdIcF_MRK-UqX;BLnE-??~EQUgT~V-P&VGPF6^vks?!3}s0OjdL=R9wfpwz|E;GyR{ADKvpC$7e;z}3oQvVhlPr4Z^>p?Uga6pbTXiI>86 z*ruwzjPYlP=E<_1&B2p<{AOvUyG-k~PW=)6C$SV%Ch2VNRz24=@b|Qb7n_NIuh39L z4WhRLpM5{CU$!bCK>^ldWshu`X(^6OkBLh85by7kbrUW-m@1#JsiX$lp-S9LY>Upa z86Q!u{ku*72dHW>Nm+4?I-Pg>OdZT_Ru ztw}&)>9B=Yw_TM54;{Bgf&GVWu{q=+VOn3^#^DF;-Vp|BSUIHpkVVxCg)lj?`xx0@ z=6#$-&3+fAbu+(Tq_b?$=+0WdTrwEt^o=4#B(~FKe4N0T=OQ*E4+qIh-b5)%LZm*0 zCEe+%#^Zs-nu`q%IN68m%>ie!=e!%498hMRT?d9tn3xiu}5bxIyYubFS-=0R#MK# z#gl*==UOWzY1G7vhyxGlYS$6$Jhrpr^{#&9=QqBu$tnp9%+$xIzI+(;5_~kR@N9ao zemvYXkHONzr5c;@WSe2oMjtC8Xlj1N)0u}o=%M=h;3V_8tZC$=H1aFbp^~EPBsXWK zFQkJKn_u?M4A#pcw+WcMR;yl+!2x`!a(*lMkcLI`0EdevI;+iJKdLVpZf6TzeE_*It*- zA82uJ!)zTCaa=l+%G0OC9ekC!f>jF$C;!2fc%A=&H&tL|uRn)1be27^vA|&sEoyuD zzrz{~;IQT}6}b!PNt>Grwkk#)j9M|JQ5(=qV`7RavmHJr>2Gpy+O?}VigC94sVha; zzaBDZpax^(xsj)?B_ESWYm{THO*{D*Sc2L76w_IUq`fds?Pq6^xO>Qb{?>c)K&PQI zEsk(MhnI#~Q+YFDLgieW?ar0oFXX7wnC}msszaSu8pRgc%7@B-Cksa#?)>kvMzahB zw^76mxU7+2GNIhU?H-H#6wg+0jL94sV(@&`&`Ug?`0@9VnVclV@Do^S@t~uf z^Cp1pCUYOlZX%#CxiB6M1m1UsZyhsd;-OrL%Mu6kWup8c^jY;(83lN$)u_YI83t3> z(9$Gg2NI4hq#QzTgs=g(T%Fh5AI~!%GcO6e-(P{m((aA%mH6#?2k-c1Qy{;d1rv@Q z6vrCdQLs}k6l%2$FI{3tdpMqUUS`HWi5Vl#AzV5bp-C7wls@Ft`;~`rjkqz=obx&D z;y*h2mFH>|KQO_$je7R>qnr1HbZ9Wk;s0sOh^eqL6^&&N-vmN17eyH0xKOzda!10+ z91wnW4fxOmNg4=;H%%@}0|fl!os3;9xIX%05E;aidlD~-ZLpgF+YFnDVuFu!ZX4l1 znszW`R{`G{j2X~-0DSLmOtD5KM_EnWl@6VzWe1Y}I}yPVX+Bt({g!57QOrjA4j7p^ zD}@5z<^GlmS~TdHRT81`T>vk&=7?0Z;txY?D~IzL#0%9PX%V_f-wT_u{9c#Cj|EIOT-paAQZbvjtU zhBEb^%FOhlMNLhH^EWQ7-___N`07AqDd|~{6h7PZrk~zH?O$an-Gf9slr5?((=gA! zWx8tJ+&F|d<5q;uxxMiV80w+^l8I5Pqrgw8pW9fy$*uh_GIA>5hhII% zJmST79pw0u4&q!=hoXJ{@+HN3gunJ4ya$3|XCAZ@=fM(eaIaMDYl~E=J^I7WADGyJk6r;7s3Sxu8V&O-XLeOIX*w$`#aM@EF_s z9_f;7OEEk4$5+}>(Faq!$-V{&gUu4n!|D>{t7jU-n4 z1}s3wjJ#I7;Id5wv~M(9R#a*b!qjqfDtCD@ta;a;Nkjqu$xlc7GEt-xR;S}9E9+z! zD=7L`r{EWELVRJCF#rrLgWWg7@E|Z|=uObYE_#Q?5n5818IAf#=cOc%q6pRY{#cI> z*c#(Q1)rx~H*IaKimieg>BU3%eneiq%&ETBk+U-_su3>(rR)8>1fTS#*rN`E*kb`0 zr_^8s)*huTU=t->hc#VI_vcXxX7I2jO8`7!w#`qC-*9$x27!Xn?L%?MqtyP|Sni+B5}?@F8JZ1#VqIb$ zS@G}WHvi>%3pfUx4Z>V^ZezA459$eNRvd6)GtrnFtMTb9Ez;gVT8B)5&E<`Vk|Uv$ zVI>xg4)sXSD>QK4c2eMhgleLEI!mpJ#cvrpyc_&JYL{WrA=J->wnl_R;wiI}Kxc_6 zn7ygFsPUIG&{;Z_hu#Kpt99o6uZ7Kg_cMw>%UK?=?2tBgD3&fx^${m&jAe^dcaw2z zBvCH2An_PyDLQjYyNE-pOtC|Qo^z-Yq@-5|2Fq?wSNsWSeHOOXG6|kS8Z|ULl|%ux zYc?^wGOQ7~MGO5&yvqwM8moLr{C?yGSV3~Pf75r%mm`@Yump9Q>z@yiBaKXrU}Y=D zu5%b0c5RJC#SV?MVF5>qXal233ljR(oQeltpz-8oYEjOyFF!Vk2P@eQ3XA!q%m#&) zx{|>iMPYR#`fvBv$@MR|2RKfaVl_0sFz*68Gp-rdScS{yl&eeR8Qt*@G&JT1?yUBx z*wP-Ep^4!@XsKXDh~9AMprVQJ67=g2lD;mo0>S*JF_5^v2<))QS@`?}1MUTYQD9XZwvrJR)3jwz{M;Yn z@RD|*wWJZ+rEIY0#;LVBSyczABG3*(i-G>o1;wDUeS6k!#xY!;3W%tcf*Ou7qs4Kj z_!*a2I`>8^F>X*K%XyYxx`QH$Zrkige#$?j@dK0otw>CL zqv6QoAwu0ht(o05**)&s&(@4Cmkn6SM!X^^Tam5qhCMS%p$|F<3XgW{@k*=)%~}9X z^-d`Q1_PF+l2Uw&DF~uTB?B~*0^M6BZz9@DVz$h@DGOMqFPmRtFvzzgayjXIavZIl zcothGIbByPH%{>$@go|oej2(pi6~pCfY+f-n|Pn#V}u-BzNDy!FH%A3Q}|}m34pr* zH*Kf#->_#O)f5y`GfDAB%o3J1eRGm{sSppGSU+8+Erh5fT7OfzWm0xg+GeDd%$Sow zRm#WDz(oVH%aX4WCF*$V6$inGYt<6GI#8BZRhAT6U}fq$cxb!R&tl~`pip5;?ij>q z%k2f6N+>O1tQ2^KvR>+k9Bpnroz81j$P=`jsgESJQY7{*uq2z{!%CV`(@FeX^M7kF z!05ixchd3oFN?Vl?tPJ>p}VdQrvpJ6o$5yd2Y;lY{(I~gNRaXXkC&2vJ(8F&`K`+K z>1bRohIeL8jpl{KnNS;@`;2DZ+L#ssI(gka^N~7Z@uB<58;O=9OIoDU67{Z04bsEBM=j4DD5)?Y!@tRM#sw}b<5Z&!ENy~rVZB^y?q={ zKh6{!dxmpzRMA_j!Edi?p&yKQCVHD^Q3T+G5{%EuPZ1kGI6ym@Q z*Wj~Rc@4!``pyjtVltjDjUv`P$C?%wCdrwER)MzyRS2wqxyS$k1Bn6H-#MW}1^j`j z88H1d`e@PiCc2xEXRU)&Q^#PicvfW1%L&na&$yW)8xVX=CY!(-(I1+oI({hCODEE; zkI@&`aihr420g(Ht1F5GP^`#?P?iey+5CSaPI8#d0D3XZ4}_~-cm_P2nB_mACqS73 zn9j%cqh9%gZe9kO2H!uprTXtOzf8;$1BWrqO>VQ|=bA?!-4D(Bw;!ip?vRNtfTNNC z1u~ZP9P$4n%_1`0NI(l3rxRs07Ny-Okk0w{rH^p{k=oK*XfZT zNZ;S@i$Aw&tEPNF1du|3^|DPWAAL##mJZ)_y zi1S+_Ruw|1oj^eu`?sUFZ#v^NpV30_+=9Oq1zxfs7JT&;%_Fqos0u~FSXRdt@I4O$ zc|sU(4A(gzVH#WlmBs}2GCrrK(%o5<=V{Ku&Ju(@Lc}4{lJCU{)-KubK+B|{6LLLG zxw(EsoP7F3R8+33?|6&&$ORo~@-IX3(#e3d@E{N^Wb-&y>BnD>-=)P#&NxP03!@3+ z#elS0o`+@w1M;~e&ZacRyReL~CdCQ^vh`jf#6Rp7LAE<%vov4q>Z|pZ96LKID|!FH zYwMU)E%@FMSancA)dJeBh5@h{&*QlA(nBlq0gT#tN~SE%Kru>-2psh>6DCOu?JmgW z4Svp*yFobwEcXH25nQ}k1XXE5Xr5EJNq$lRUsZ)K;s@XkKbJ0_Ny%_fphUp%jEi`7|j73FCH zl2L2V+Xks}>q{ABsB9WNmRteAc3Y+q6n_x%*Q3+dM?bwe9w1rrJhQ_Es`9idZ4@1a z+6dxxq`^w!jh0N7YJw-Ww8CG0YLhoof|I48j28zIqYyeGM@chMPUyGx3f^ZOOWG(| z$3R*}nsb9alJ{0`K45n^7_=dy2`@&JN5Rn?QkmiSl&o)Xe2bXBkc3LiJoK)u4IT&N z*V8xXV`RxBUH=Lv7ojg9s7Q4ZikgqMq0XcfUR0R?+W^!%L$xd%kBWjyS)z`U4#MGx z2k8}vdoVnR5$1C{lG4?lzklHO&>drxUNHxE3eew{;;)OwRLTj=V8LMJ2ZzU#sp)vj?{F4pqQkIrC?F=XD+2luK9Wpu&iLAGv3^^ zF(5tNv|RU2)6}%+s&!PWa@yBuwJ_lAR!^38PSJ`w=xFI8dm~kS5Lt;tiL>}rEVDg- z@2Omb3A+|7n1vp+v=(+TOD4tC!p`_6+yjc@V0i-=+zeS11>YC8U|L;8w@pscP;DQz z*35=ch%j|#NJb}TWB)522TeQHH^>4S(jDLKgEZdDayDjpj?!zM&X|3ob7QlgGr<#I zv3x;du`T3^M;2&Iev=2Hx-wcm1l#>yulMgeWW=Tsgm;kssLI;|`Adw(q>ziaGFAfp zh@?_J`EAM)!mp!3=4Q{+S=q&BzN?hI*hniPL1la>+E$qRF%XV01iYK>_Rc|9wOL zm?PGB+@Y~Km96_2?D=giPo#1m36RF8z^I2EO{rjW)KQ}Dh1wju;?n5XuvFNP#Tw%H z>=F!cAblgK>^|HzO`FEYhiVOpC9+z|*|mv6a6nY_1htWPL4!+K>cnM~NRiWok6F$Y zp^=&_OY?P-hQ_f3j|0Df>G~jBvCd)paBbYFbw?{g(!7uuC*uQ^%D>??IyiB^nIV$X z@^#pu)qD^(FSa+%yUcnEwoFAHG3m(tdpg2HN$f~>0m|)?Wxi{QEsFbPK3=o$=)%!e z;gD6kRc=(V#w|gm)zF_v1GJL04pp-cIe=Trj954Nlj5qSW|=0&z*t-!j7=&-14dxO ziEFDWk0kOIV|R{IBOkWdKIQl<9oLgd*CAnD?Qz$@?zahEFL79!H$OSVwErE}Hqh%H&Z1>yKu0|u6e?m2;r3Sy?2QJOM6>we8GN^VHU z-kA%^uG#g7mjHat8)YM5I-_IeV2L~4eq;6L7g2Oi-#30eeRB(T>|1mfI{@vcqLX{p zb&^2fb8*>a6QVr(#4B;6vwX1Ee|F%Lm*4@%wMU%dBD8Hu&CEKNU~KEPyap1g3}Or~ zW$xDUHUowYg2ym!erBBi>p%auiuxf5nYv;sCtAP$;@Kwh(n;vJ&3~VWg4y|@_CU7XY=JsTr48^a&-pU)GIIFPX zsoQ`hN0N<8Rw00utPa%lIMU?7+~YhDDTZDR#kXN1Mf_Ds<7$ z=jSJT`Vho})VFxMpNk^fvxezqo@UC}(F!}2&yxh=MQ^{NQ7oqBY#s!Rg)H1r8|^r$ zGTAi;&$5Uc*J*wkr!?eUfda(ch@&wJ7D3FqZSgakQppL7qqt!CDI#X|;1$HKIF&x7 zmUq9I>awRGhP!~VBKsddYKK%_J8)yS0h`N`o*xKF>~RwzLmu`yopGBQNZ}3AbQB9h zzz)GMNRft&fTLNB+92I@UZgWB&iy!E2>w>d4=uToyjYIeHQ_P@h|QpaEtK@gO!H_G zC1$Vvo{nIZibGfejyR!LG>T!FskjHjVKK{whL#S0Pe(&w#`GOyeSXBe>VkHsDDfpF zE=Bfqh_4%kW^MA;7Mrax@=-Ej+^|#iQa8m3eN72gNpvO2>sJz7l_Xa3@VX;;mD(^j zJk?!sRWoWg7FOK_el@bH4KQ6>Qgs)o)(WcHZSrEyi*y$K120x+6M0q$mUyxt5x9tX z?pvcQI!z%@z_0MxBP6$ft1M0*e@!$I8IZo`*=YyAfCS?=M&=x2dF3ofXET;y=moJ+ zrD?8GwVFs%6n!pKcrQV$IJ(5}AWA_D&=yB=6qulqbFM>AskoLoqvEU%rZfo{SDMLu zo`4U&Bi#~5bsQB1WQcDYRCX~=80F+`ltkk6^HmhIFf}oOWKXPW^8|ZaxDhP@dMvj2 zE}`M?a}3Nh6AfsK%n?DVa+N z241YNhl<0L9!WB~LByq#BSkn7C;H}R#bi*LZEbF#ESYK)n~kbhIHv_(E|zdh)cT&9 zbh%PXD80i5k_+U;NuG*1x41x^Hy4$#t#{|-=-2a~e>r`9{Yh!h4B`T1Gwcd^ z{nFg^J)MwVR%(+Y&Z9{}D4FExJOh=Xa%(Ec3JormujlvOnO*OSTwj`57b@ zD-|KyLw-PBCzX$AXj6uZu<`8fh`|GkL{PMwB`T#WI=WLu1ueL5FH6HC*Q~QmUPT{) zT!QeP3#R!m|8cAXBFZ?1V1tRTI%2)^fW|B@iQG6HvrdtA*i{r1+Ab7URFP9lO}-YQ zmKb-#t{NKx{nt^&N}+8#Lu=i=CCx^`B2SIzM!PcDN_IRYY6Z&>)sJH!DXcn5KY9|r z6iG_*pw4CT@?DJ-$?okEgC-ohFtNo+ZQOVuh>$C*L;C)nzwbZs#2C;#m@@q(;e9)% zP$x`hG)jDF*)I#t7Jis;AM5&jJ|0IOy*5b|A+zmak}xf}T+pz*2c)&va*D7p;XRiQ zI9a))G0|d7LdkS7$|D_eSp zrILamK{mVi7*aL4*q`K43ba0^oI_S?2u2oi#b32aRxX0V>9(UGF)mUxpMb0i$R88C z=_sElTP}fTr=yL;tPG6sKktB+P8MtdK5t(C$1&OBf5hKyRa%9kjF-H>%r-NtO{1rg z`Z?vrU%|`%EdWsns+s;_rt*>hh)Y7n3K)~8OB424;B}trxEP;*icofiu|G>ae{F}*z;rH2R75mZ*T5_d^(J-;qadW+ zCU2rEmIz)h%hM68qgcK<5ibO}9MdSC=j?o%Gd@k@a6tAw0@9$rutM^;MUnl?Xs95Q zs`Z&f?Inxp;*14p5^}s}XDp8(BNM*ri+6ku8X~L7zPPQTsMr*R9-J~7Ms62!_3F>i zhkLgFUf=%TX9}VsfxN`li5Oi@+onP!8VYKp zP%31vjDL+rI^4u{g4U(YIX0t87(pe-X;lRphA7(t>Pz;YS`NIs;{G8=6%@f*cyhSk z**|!`cj$YMyE2oq3=-2?=A)qm9xa)(U;8k8S1sHkz8!PaPS`9eJ8hy2VklS1P&sO- zW|n0At2I{F)eL&w){OutY#T8z5i>t$VthhWC~%_^FkO+kT$Mg$mpn??H5oj&vSBX= zib=x-AM!tVnqY$04#5Sc4TgrGV;K%Vpx+Do2#L3KtLR-~MaUnJE%9BT^5x5xr-l7^ z%~a6Dly5vFDwVe=c=3~%w)y#0l`5*BRjK(OiVSFh5zhc88Eoj_Y zqh!l%`f%N*X*d@D4C8bXCBq<15@ZC2l9}PH4azEXJL@(q{cr`{AG)SWm3EgD-MHD1 ziOcP(7VArG+!&DNgcU;>IdT}M!DTuxD(y8E+_=dx3sGFXs*8;UH*PYcoQut}s>P;q zH)`-Y&BI|Z6*LvrAPei7tCqTTV_aD~tZGje+_*_Fk{c?eO1T~P>&C#eFWUZC5 zy2d6*h*h4^;F1-5n9d7`i8h4L`F6ugO5L=voCPepV&O21ctCS^2A3DSb(2f`S1NVu z#>xjrbD>)|HBT;+^g3D9m@KwQb78`Vs5l%7MtnG06y}gwUaIDd{!EHgFimv~jJd8@ z_W%frBn_@9*TFwc@)T*rvJn>zHCr`IYRB>f2A)R#txUc@D~^q{{=G$7!Fs zcPd9pZ6%B4Q(ltQLK)4APC8y6QH$YRw~8|OeItj-SK|)~o?7?rDxx|R1*B~FO#Yas z1uIRYfvNH|C}o#Kw@Cj+XR$%Rs%KrJY(3IS#<~Q?SyEcja+5_>9<@&Gu%N7T-I*xe z7^>iuE6vKjx@4g!Srt_C5eWLQY5#4V|2yky?M#+i%$EWN(+-*Lxx~T8=n3Fu#c0@a zdS3x93*SsOQ58*m#|Pi>{|>Z&12`^9CW-}7Znrf=$E{P)$k@XUxmXxqGqis z3(LEii*GU!YmE8V9`ME;TiLuvI<{rSdrvA%tq6(`SbrWDQO6WA3QLfl2I*`B@hVYA zuo*4TP;2QyMFpxlw6cIb6@Fd*q@_btIyJCP-T9%yEo{_hS*dd$eYXJCga$~;1F}ik zn+|b#v@x}m&I@Xp(xOhgWOP#*YWKU30Ko9ox<(+^Wtm)y<{b-cVB4D5HOqWjd~mF( zmsT{k&Eman>AbpH$>dlAsF)(J230l**08F!!VSQj{qYK-JMoHHq)%Qa*&Ko-ZG%9y zh3iQZKY5m0Ra(UVq@%f)KYrP{0cYU=cP01RL7D`#C?65!YNYaJ2Zio(N3lQ1cCpCV zFJr-owPrMXk7}O};Kei+%m33CU_?5^=IZgDXJDxjEZKhO~U*%?F znWf>kCsf9TeJxtGtoVJAD>H6YQf0n9kg2aqroOcX{m=c}#Qp=#=hlw|xg!2=?{IIw zWdE`MwEy(M{^LF#C$4SZyNr@>AQOwfrI|OgqB`74yyFB4*+jl44cb6saoEs{lBXvWs``0u&;_FHe+J5j+6nVhr{<`Ppm-{y zxCD*wO*ID@elW!o-E(3jP7J5f4u96v!*vc`e%#v@gonu^DY zE?eWNVlJq)&?DD?+%rCG%EDhpZ|IDpoL$p6_Kc&ObTn9(1V}A{g^qZFbT*?&IPh!- zx)+1L|MvRb%U^yw`}-M)f%o_RF5}Xjq}MDzo3c3m`&HlX{XG-Ex%lm~kIaHX(d(ww z8Mk&x3#0n;ZnZ;x#`~=gR>dpZg}{0b?o^gvg9Pc6P^49Lx>GJxna&zJetUpcub7%y zttz^1LF{ponwLmTsO~#mAX>7cLUCLB7{%!(PPbbVRzPV2I~~c)jiZ=7|L(;v@6L~2 zzdJr1o*lh=`O`1|bNKd`U*DY%Pkwp*?)qC?786`{JWlN18{BuYz0wcg;qA z9PxrB9U6u?bTA3(Yp&` zc-Nd$wbL%$mo}@Sg6Iv%`HK_lTg5-fJXJ3nt2L*dUoKj1-*L@xUn?h_(kE+fhr8s_ z6iV4s>mnc<@GXedF|el|0csUGDLvLAkR@>~REpNF0;1y!cFS)!n>FRJ^yw3sXBo>^ z^d5_s!PNRWHBMzs=`z5Qe^etVr(79G)uE~pPPu3qRM`uw5s+KZ1yl7hGtbhU;8y$= zTcL!(zuI1C=>E+%LkZZec0&o)&9_4d&@J{u3DU9+Q91REdsS)U=!{Y2#cy;ZE&gP6 z|Ek{K_un}kQ|Fd_vEmEdOCN4JD{D(895a;-tYsQHutPqRHROLz z=q~FBx6{Epg&|y5so=@$mrEpyI)IlE^XdRy_E6nca;c*onKC8-(w(;C#3a(i%CU#; zu01<9xh_=-P;}$gkPOyH-Wr9BN@Mw!W-i^ljozoyN+T2Y6&iQO?^b--I%HADg>gCj z5{*=)mJ+hto>l9h%Vt@zcp2p`x-%9=^VxvB1y`**!0=Bss`WhII%uw1y$uy_GOksI zZyf}&nv0zYWwj=b-&$k+@Z9Bd6Z;R#v~ev1kQMe{hfnsNl3Vgw~W3+JCs^AeeGD5GGE!%n=LtN z%~4FMq)(sBwAA}^wpO5}U}1CawW!GgdMQM!4^q;%!g^-S@hj)Wa@ppZ5j=E8P+F9m z8aMPdZi27Qf4$1is=Wpac6EApZec*R?vis7dtueOWM3N?6Rp{DDYF{ZEvH-Cv~2+& zeb0|oE**r?S*}TNEoIhSZYAY`rF>XsRkQB+IBc$Fgzo^dP7-Enn!I6mdh=~)znpu^ z!lROw>mKmYX}hf3dHhu*4-I?JNfBnmEKUZld%y$JVVP<&J=Q(6tg>TS>s{x|x|=nx z^HCYWpH7B#H!YM>Oxa?$QtqicY|rHrO1Y=*rX43|mrd-PR9<(}mYokt*R}Hm-NRa} zWDqhCs>y|JgO8Kr%i_`?Wbj(tI@C1)OG&pZkPsPvIYg(j1+$z_YePg!^>1(r=5)sG za}bhr-E7?dMI{_Jx&Na$_r=;?U@QH<_RIbs2mRiYhx`A1Jon`PWvKxE1^vK2eIncb zt0-pN|4W>Xj7|CD){cLq7)-6y1G0TjROm+gUyjqd4xlUc|6cF#up0mS;9>vY$5Xri zr8CQ)-|?$q*Sq!sH33kRz;kV}=&h;AJmIV$zC9w4Yza7N2q1{$X0>?5h^i_L)xp*o zoHpsg#xkFh1QHVs*2r7L2PLh^*ObqYvhBw<#Dm;sf7F9-y-P!OlIBH=?2zpwEyyC?6|f7jkfdZ;}X^$BQ>mPeP3sFPqXYqzqXtLjIq~JA(ksny_wwlTT`uv z=dPYxoc}VQ;%YWv#rfag?^pQ$Cx`nF=l^{?l4J^ksm-)UG*GcC8{A&y*G@?xSY{w( z@muf_b%8hk>0QaoKEii*ov5*tb=$g$tz(^)1I?TwCOyYNjH3MLI2mb z0a-!+4|-3^_8$iadk^xTdwH7eKgxHZwTwSzx3&C`C&xdB^~bj~{h0j|+I`Gc8+|N| z(Af;KYW1;ty zEgW7&F{`z25Fc)(H=TV;^M%>h)!fg%uG)Tfm%8m5t?<$f-JF-s-QpeH!qSv-_{hX^ zTi3H_xa~jQ*|NUandMZPmPdV4qCUS|L(}MQvP+UEsI)E`{=daP`U7VfmHD-|8XCmJ zUiup*QX43=>$~x&|7dUHu~A(QIOI$VXgtjFG+>;kId2#)3^U(=`78}tvyDQ>>HNUJ zaPZY{J7hQ%oY2sm)59zchtrf7!zf!>PKOB@LPD=(J{wulb{xD_(v5#refYoHv;O_h zpR#xsO_DTc8@91Z|9{XwDBb`1Pxl`3|K7{fCMUEgSe{_QaF||lJ!J_Q&7(Msk_pLZ za7ibO`(B%zPa{tFJOk|vf#7Lm5~m~NlcQv^3*@edt{9l5TJLERdTo+0OmI%Nvz(2i zk1WK4_`mJ=1pI*01ZoPDNXBv!M+x&i|K-{6tVna_wTVoQ_170?B#d(I`ID&V!oP^V z=a2r8ci~_4Vmj%Hf7DNYm2?fD5e+Wq8Nnp#@y|VnL-+S=? zxu2&^PBiOFE@Y%cq9*;q^Q_NGE&{Rdr!)_*X^y$~emPTV-Jp<6zo(#G3SK1UyW~2L zih?C1O2}VQ-1k{LpG@c|W)c9=(eb$NdFLiu_b5#ZUgR|6gj$Mh^1h1nhwW)mWP@&Z z5*5?=2!u?Qm?ubQvw0E~3$?_Ds$)AuM}Z}2LMVA}GMQ_jj;v&d1o51UF`08gD~l;3 z?|)|T42$zYw=0wI?I=1*Fo9m~dtSRu{=y1!Y7zmxPR6O{U0hrQX~NT(K?*;?ccz4f zA*t&HQ4Q67^yLslDd5qOd3y|03=S(=^t|C2W90pX{&-R8L^n(W4nN?Zp(xeef%PDx zags0%=7K>6D(?@WldlC+8OH>Y8&$g($6=ToTotqY35;1nY5RH8;hZWl?BRox~}n9*?C=ZjqblNBkp<7*o!$Y(pmHmwNGGJ z$WLr6SPr>FqGZy5MU)hI+J#*<1dQ5&f!8Uf4A*vN1n@eMhPUIB^OSJLU=;Pj5F{tf zP%8(c7P10T4nwcRxbBjxh)0DP&Mi*fU!lBUL1*!YZNF>%+}R~*PV#voDsV=eC8&mq z531{cPt$9Rk;ZSf=5Is)*FXQ4$H86J#)|WQ|4F~1|9SHC;rzds=h1PVr}+^GR!(v@ zW;sg&MxK+c7im&N$((I@-gurs0rFzXg3DKO+E3>(gWBX1$P(J*H%6wBpgII{Bl~z= z2xvm4G|Oa)IZ9CeK+-YE8Rcohccnl~P=*L|8Rhd>Y}lO9a6#C|h|BOH(U@#M0_g91 zA9iHR>Vx^>AAbxO>~8-Z+(rP=7iq|LNC&LL)+p+qr)QAsX?urs*dL^Ig!mi-I;|bM zc^rU~oL#{^L1u{MqLXH8kr`c(DZMg1Lqm#?6k?#%z!EXCCCERqfX=y0^%RK_tBwj6|FyDM^?BpUPtZPK2CE`;DmZoo0T^qk?sQPI*CEFIm9?fg1EyAn%~JOWvi0 zxY!Be`Z8^4IA2hI@u?9oE}7#5l>IDcAvt3OH1KOiPT3#xNW60zYdOGbX3qJ}K1K!U z8-zUk_lxJ*XdD#Z9zHt=_j}*>Y545flW{k@oOEdx@vanT4yDFr zIGfB0f0DM}^q)NQ-m&Y>J670xn{-}AJd5c9)G{`Xm1C%L{^Fz~28eVvDvWf8V4BK~ zvT7vfFHT73O&ZV`Y$>tqvD8DK=W|9n7_}Qj0z^Q$VkjP{r%=w!tNX&cG-UeH;LbZa z!w8w`_A!#16Tx>Mh-(pdm;vkS;Uy~3A5^P8HO(_CSfousp z_!cSXNF{PpcMA3~i_@ItcQHG#aMzw5S$e(sk%jMkf~>2jREVrMOU^crSelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/104.1.4+up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..c64e70d302 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/Chart.yaml @@ -0,0 +1,126 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 104.1.4+up57.0.3 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/app-README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..45e3df9325 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.4.9 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.5 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..587a5f22c3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..32f406da4b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..59798c5dec --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..b2b1ac4a4c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..58f834989e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..d96f71a9a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..42e0049cc5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..29f844722b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..b3f7e373c0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..515ee6f46c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..9e649a4ac1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..fa2a8497d5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..f0607ed45e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..eeef424a32 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..02345baf58 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..7b7807944a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..0f09aad1d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..e3999c6ff2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,24 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: windows + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-windows-exporter +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..9b4a35a543 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,367 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + os: "windows" + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/104.1.4+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..2827e81196 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,467 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} + +{{- define "rke2-ingress-nginx.namespace" -}} + {{- if .Values.rke2IngressNginx.namespaceOverride -}} + {{- .Values.rke2IngressNginx.namespaceOverride -}} + {{- else -}} + {{- print "kube-system" -}} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml new file mode 100644 index 0000000000..3382ad5b61 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rke2IngressNginx.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + np.rke2.io/ingress: resolved + name: rke2-ingress-network-policy + namespace: {{ include "rke2-ingress-nginx.namespace" . }} +spec: + ingress: + - ports: + - port: {{ .Values.rke2IngressNginx.metricsPort }} + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/name: rke2-ingress-nginx + policyTypes: + - Ingress +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.4+up57.0.3/values.yaml b/charts/rancher-monitoring/104.1.4+up57.0.3/values.yaml new file mode 100644 index 0000000000..97019d626c --- /dev/null +++ b/charts/rancher-monitoring/104.1.4+up57.0.3/values.yaml @@ -0,0 +1,5433 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + networkPolicy: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.4.3 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: false + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form

bgmPOY@UxuREiJzpl{CH52L{d5;JwUQQQWZlFpg!C0ZXexHL8i1jr+~QVsqj zzJpxXyadm?^i)!v7r_QoP|`n}Op&j5Le;1b_7!BgznK^VI7;BxLVg zhHOR67)%t*a{L=VDlJ=eWKdz|4sC>qP+erLvhh=uUQrYrhs8qUi*MaCt&B@}T9D0x z!&9v8x;e14#?KMg?^nK2*PHc?FUCY*sJpm_c2`dgXtK6_94PhaZz{?qm_8>e*czpJt?W15leUKkpdRx1O^4q)k5TuU$iJG6JczY3vY(}|K{JZ_rG z1to_eu;RV}lV^2+Uj4D7A&EM~&`4ou4t>xde=g+zW8<8fI{_3WAKSKV+njh}+qP}n zb|%(8wr$(CjjeO`VfR+ur~V3E{p)T{adu=}rWdMyBjQXX?4hIIj4#6E-$VV8U*QY5 zS!b;U!>eG&6@8T6EA6Z5?cyriO|pz|*hk#OYB<9zjXo{&ACdh2R>B>)hlX0yl)Ov;pyI`B!5B9Eo;+t^rbZ;MzFvCuzu~mK9Q36Ginw<^%*pm{~OtIFskwyB1FL>66Hs1~$>oW|iM9M|RKgNz^QaB+b+9Ef+^*{E5Q(n5K0 ziO;~He35J6{lbpSA0bQmZHB-n#M4@oAwJU}E0Zc}(NiFyjks!dcc0<;^b8}l$)81f zlnep@*PsedMuET6J9jywxZnbPSt{Up&Lj3nbk~!o{v34bpk)eC{p`YqsC7HodfhXQ zF>N+F$R~jVN#w;mDGI{k(oQ{E)=AYB8qDDeL4}-%u1%2OBzV*0vzgUirvO{uK6k@W ztwe3x_C-g#us)};x=Tew0^&hB{oRJHF$@1Mh0TATI3jT=gmeRV;1N-l)AGLqJ)(OI zm6x;T3~vJ?8%qfRb48GZ%QC3*YWLrD4$69(hagUQ9mO1dP;(l}1D~#*V4pOp6hOjQ zGgIKRPxbD;7L3`JzLM;{hW@A0lgkNS1Ugv2A!8k7E+Q6XWbGIaE&gLF$f|Qn?}TpE zSKo>Fc>*?JZx<(%ZPe{dbt_q+Pp;F8w~Pqrz#2>u-=A@*K9l&{I2?K(JxP*uufinVMH^$YmWV}k&(E1(n9G(oJ~hW z1A4iB{`cok$CIm=nE5@P-}?DB3;LxWeTavWvJ}bBCXF|~6v?jk+VLih4ASj*!E&x~ zx2rF(SpYPT1o9<#^}D~JB_>tCTZ6gsegUh#2$5g`!VD}lwF*~(+VHxvCxHo`kF_;M zD?xWMD=E=L+A+9=Q)`yoMiFm6j9mz0ETdcMm^abapA#KIuU@9QU4HNT&!z6pa;qyY zwsNa0FA;u^t7=$@{K}+w6zTR1HcNYM|-dJ z9a9-WcLGa@IV>XbjN(CO(Y*PnqkASW-ySxM;<%dgfP`ccxClT7|JL^jqpXOq{9wuXyC&~%4^nt zs)%GkF3IZUIS0tWyQt+2OiO!O$FkTC2!wa{TCQ-C zmjBjncg{_O8cC@(MqBF%kx6%LUZAV*sWl$2`YK|ysr^-o-Bet)d7QHxmCnqcuhs`l zk5x}@XV+W%Fb}v7VNuxc9$kB{eaY?g?6eS2{{h$4+ZMOW{>Q$%yKx8kl-=)ASGvHC za{a@l=WwKIGO|`Gw*-Kk78fE#M6O6DcDYqyiD3TjE8t?R^m8<zOHOEk- zU%PBoHOv{jZP^=!AaHK1-l|B``dM=sQ5B1W^fo!f-Ll7S7pA5z_b64RE4=TKPDnV{ zek#AjH#FQEmb^TB$?Et0WP~PM={|B7w1n&dWJ@KB7K)pCiY#_R-W>X|h-J8`ZAmGw z?t&PksgW4aDU5y5e%~B&s87J+i7c3G-zJLuccv(XeQdNugTn${d#9(Sblc*J+X(_B zK@*ifn%TiRzLW)UJ71NO8aRehDQAkg*(Ugl{LvG&+FX*AW?V)W>}zZ^?TFe~(7E;q zSV^4HG#I>nBrI1zpCu>&g_j_HFM?QiWeO<1)2R%U?*%Jr=B91Ike3bxs7psL$P_#( zsdA-@t=(S^LEWM8!ZUnJnNRGw$2QrilToN~J$n#92;AGqUR)1$OQ%y0DEAn^aZ)Wt zg-EsIOXS{0vgE33R!v-tPB%9vGu8qq<*wP77wd+^VztsuLaCOZP}8cCOx|TaT_hSZ zH!28j{Ce)@T~pR=NKgG5XuFNX3!cr?tzL1NrMvn zy1obZ?Tbqn4&*3))zUHI|G6Ce*Mp_tlX^I`Dk;Cw5d?*wl~xv*HT;!sTtEN3^|9hT zCcs9jN?9;tQ2afenR*26>udhPJ$(V%wdvxmzztjK-j-QvsL(zcE|uAbcV5eCpADH7 zU~s6d0wi{lobZ8)Wx2RCY;G*UI@!?zA~a!b-fHey$Z3~PsO!u3=HcMlRYS)S7NpRk zpzC$m_pn*V^LD&4wEPBl13Sk5Z+zomvM&Wab4Xb%T-DYGb(Xj+mbX%!jFYl!j7w(R z$k<<8HlnPkWGOev3zWo>L0LR0Df6<5-+b2X-zciC|8JN1L&xxP(!MxeR*yQtm&Ugo zs0~oW6W^&8x8ZNS<{E>y%XR8{H*}aSpOl2>?s1zc7T6A$wO%q$nFHt=a1Pg}lRr$a zuJ#v@R300+P(95og!LesG0UZE`^~cK6h1xFbQF7YsoxxiIch&}K;JlR-5kPO|H~fR zjx+8qCti%+twh+NK#>(l^1*Hxq>cO#!=5^t9UZB}j*pNGA-C0ivWanQ4T;=Alx% zUS{47=>?F1!M)`uAXd1vGl6R)N6&ukE|&sO1rN}fW%AW$#P@!Bt^jr8`Q{Az`#65B z9_DWg`nmh~xI7JV+V}SG`G1oN{=7pf9|8IkqG?jtPXptgFWY+4InkU&O<#Hc0fw-s2dta|^Srg_60h%}IMpkF+E|1fCLQLK$O8i%s+DTYArSs|Kx9ZOYa$~Je z8$eL2_)!f$2Y3#yW+L)+%yAh9@rohrJnNRVWJn|Ly2gb5g>>^fvmk32pUBmrR;LdB zx2_!8V}0z*qXR0K*0^xhWg@d<_-~v31*|R^WapK=KB=Z{p<1N{4jBy^7q>NTX0t2pUeC6WBd3V z_hDp2e2rT=0c0t$i?$?aD_sYs*7S6RGq|6MLK5hZB}@?*_o$b_lnDm5UY#A<5}rYV zNTQ0n{)2m;WZ1`L3(>*Kp=)nw0J>PSH-Dzb_HZ~snr67r4sIsQk~eHuD^B?MpW{Dl z_>o^YG+{{6vaL2kphgT`(OJZ{NAD%~Ixz!BNb&Uf3}_A($%c|Jkf9!dKZLnVsAXtP z)T4*HRHMey!jJjX;5BssWtfejyfQR44?p8B%K=%ac=G|-w=TO(DFb-H-?IN$kB?ch z`WII>6b;H3Ht6cRuMv;MkW3~+3Ku?9vw@IIB;Kak8;KkyREnvP?bF90SuA<3GiakO z^O62Grir|j!$C^h4B+wvGe;F+R5#$OTH0URQChg$U;rMe!*qv^2b2)nTlRJ`%&C>Jl$`etUmyH2$({slRQZ%&V z?hjaUFP_ivipG`NWaioK64K#OHzf1TzgxX$)A$ic-+TlY$WGngqE#0m3~^?mb(MZp z5%|jnWTl`C=(0E{12+rY0nVXvBZ-P$hR9H)&dU4FZlK2D*#D#)Orn|1KIEBR zlc%_ z2$Ml;ZqWkrRhEPK+`Se1)b_5RodL|cZ?Hp@BGN|E3zb7fjne5V0M$OHG=S6aAH%n; zxpEQ|Gg9z22LTH%)S~?5sSC^A{5GTuLLDg4^EKmxv0&B~Z%o6*u~}!WjNISVCt(N?6nO1<;JwzZs2VJS-bry!5g1k?+eVJWXJn<(2P0oE-s^26@3Cijr$%a z;xA?tu#y!v2uZqK%a$_3gIluehFnmOkXJB#@7_Q;4xT&xQg3>e-Q6;xFN$62SNL`lY zr2;Vt9jjr6MKGGZ!(alG|ageR`WwljJccvPs#n^@m`K0shf0w~BTkbRnh;|Gp!6g1G`O zd17BMd1C%kmIg9YC9dH+TP05FcnvV&NWn&H-=RarZgF{=e-l?+6Ujv)AfIILn* zaBZpS3Nr$0uq2{nt29&HY%u;>Dj~Kb6;-sxrHG&dsk5yuLTWR44E7O8$aPX6@5$02 z(NyNjrtboi^9JP!n!*f$d))Gr_6MF;0Ih-DV>&WLm4y0wX#Mx&mu2|b*5uyi&TRCG zQBx@@ryRSiY~QyYWm=~AcqW5q(aBL!-oZy30P_ge>AAX(^OWEZq%<@@G2|9Ok$@XI`H zDUKuN^gQWQR;^Iw2qHteK`aVw=8XP0OGV_dS|FEUJIi9{8X{da{Oo<5nURSgBm*N0 ztEFl3Bi=)lHj8!kbOlQkGKS zpdBku38}OKe ztJQ#=O(b?H8YYaEqO?z7MG|UEIM2};%UO?wnk+^_gM|2$Yc{|hqIgyu%JD{0w}&?@ zc9DSUH!6|pU<#tQ@Hr08e`qsf4~6g>Jji(#oe81)z|H7N<4i_Gx8c%ZUjO# zMwBZ)B7}B>oi}7p$lJVc$*zshR{iO1#pIIo(5(SaJq#HP1YOPVka8W*nQE00r zw7pI;PB<& zzm1edeLgZX{PNAi3E=P3X(xiMSmj-9)EwE4USPd?o_Dd{n8z%qDu~g_s9!ou4!gvp{ay0wkEyUh*{OnF5Tl2 za{AmXZ6cM%rhGNT#Jr~{2jqvS>VCe84pyWv2qbpVtsn}x{iF_;A50jo73{%u+iQ(_ zZZmL$b-Rw`?0z`1b^vXi5x8m3v4p|49|LT`b~`xV)2_9ufv##C!MHVTv;s)q)=70u z;)iC@ya&ZyUkNy5V@R>lHdgPD)oj}q)og{jDVn`v%)3i7S!-ig zo~R$u$#14Gb0UHy2AWEpf{n`*yz|MFHQFm;s(#;wgBlkIJxDU_kRm7W(091l|Q&KzvV<6}b6eoz#u9dF8`$kmSINTOi4El`+mc0~f zj1aA4?!?^=g=(2pr+Tb-Mj>rHY$HAMa526XF>?w-QxeuRy@r$_+f2e9yL-=BwoDz` zIHpA7l}oO%#>ZLuD)4wQmeJJl*tPUbc(P?2H1ocJCbsrOJ`E^i4Ro1NP610ic#p30_&vt^*>&)rRvmr^=YA|Vu zL1}E}IEPP=v|#-5@Av_AttIsG>vN$Pgj%@UvA2DFfL}kp@~jz~Ik3Av2l>v@k#)zO z(fBR+vW(f& z>L-nImCk&6>3PNpTPKwU3N?@a_sc=kH}M_0_EJ>;ZAjS)AvYN>U|4Iy+UclIL`|oz zYM3vvJsg)s*{3vbjh~0L`^LBztfaEKR?{4IaOL>Cz*uVAIYS;d2!JKlSijo>34aBX zb|K6&dBXx%tOb@fupInso!i<<^ z#>|ukZjEOg9!Ak1_ANx|lw~@0G;8X0W>|r6OcWr{TgF)(DPA?)qjsCfiDxLF(>o`T zwV+})639Z%-OI}Vc<#x8ONB@>JoV{mp?irKe|l7{$S6Nm{Xw5WudK$z1T5HP{Gpb% zTDX2^yDwo%UA0$>9hP6At3nzF+*Wx-8HU)|S+VHK&Q#>5da|tMp|KgtZo`J-X6NdO zW)w7&b?V2dO~iU8(7H*gDGce?gx&d1iiEb1RQg=+%4cqvZqwfNX-!L|`}-I?N2J%m2dKTw}Fum7B)sLeR!Z_4=3$WqFeYBRMMtc@CodI;d6^)0U>T% zjU0rcSC9$I!2_4b5#vCo76V6~KXKp_=YhlL1$+=x|3ESRb=DaNIJp@(cCeK<2yr{s zbLr;h2RQL`@SI~O=QVuFdP&@}o@1LVu#UXgFR)VZ8~b~WWTjamy${#z>v#OwZyj!j++yi4sAXkKCikUw>vF)*KvP?!hn z&4Z1VhnW3+mEM6_J>(JZFu6-dDB4qJ=I2-_r^hlH5}-jLO>)p>9n6V}GFuM|nB$=| zA8-+K%`eYGFmR3$ovIA&4so9%I4p9ixO)nFh+*KFMF-kjk{{}*b9SO07o=6Q8>2~v zU0g&>@M2t5`6G^|peZs*5Ci&|^q!Y3&M3^a8nczA1L(-WB8NupD9OZJ`BY4RWo@~M z+^ZbVaSkkBX}@7ht5V3f$jU1II$um()Nw95uSQH;wrD$TbX3SN?yNxz=6E|t1ta$r zDGKBBv3eSjf1ew5_iCYi_GY&2_1|&U>MUMQNh8+<*Oh%!!`A|@viWU^hGtv&Of2gIc&^= zgs8vi-Wy!~1iOTC@6evP6{Lp^#ryyg;~{C3(q%~AUdg&&Lk{w|uWy!enoh?U^Wn6d z9nV$!5xb45P!IilB&c)X^V${0o|jYFg_~7jZFg_0Z`cOzYOc(Lmf$#gU=0eA1^ZMs z&Mv`N+68868Qf}}8(urs9|#Ya#YPt;sdld0@OT{@#K|9i8J&!Bc&RHO^E8pd!yS1` zJDwX1uhBFZ|C5tZ^`>wIKl?7^Nk_wRoBl00OA_Sp{R-l|yhG*sy1~f@&PGyiJr-hS zHwZ@;fVyP+)Hd&Y^4Z%W zl?Z-Ldg=O9&+qLSazfA|5Vovhu@wNudo$%y{Rx?RO;g3c&Z<23rQj#EdLw(W(~(L5xQec2fep0(|}x|MPgOdcf34NY%I#k?bcRf_M^JT zT74~YeMaH6+lLn{bWOJDwwuco&{d8TC>0jvBEd?H#r=D8Tz`+C18!h%9}sQBQ54cF z!4croYHbeB3Yx}2xx_pj3 z=PSrW8+Wm+Ka~IsWdDL0Z4r{;#UcG}by?@p0FN|G;wW$4$&-E_s_5(Zv6R}(EEGdb zmKuUA6#@H~F>@c?ZYn`Jaw9cy1so`&-NC*%7-SzfD?t(Z(FN=c+?|b)#Z9Csg)G@9 z^sMq92)@dR>_N_jz!UX>$YLR8#B-4DdYISXy$^B8&w%ZzkmPo_1a)^JA9*T>aS2hX zkiM{%sDtgFnbgFM@T)k?IPB%c3Z^30#POr7QVd;G9UE8J)7@^ooliCP0X&etA(p7Y zsr|`J=pz^gjq8TfLVm5;b*b)}RW#J*Xtam{;er8z24bdduS64mt3jYk{{l)U`5o+T z&Pk3jkm@x7yzDLmvHB2d@ku$|ebRrpyhEm`hI%5ls)06UrIR%Cg9BXT(Mo!(!b#Oi zS*i83{9SAru30lkEbtyG!Z8+4!uBE)|-V5=Lb{{a&nfP$R>u zW|fF)e*D7LNpdqte7JvXW2@*dje^7JPinLeZH*Vj;!_(LIEmStEq_m{A==3MRpm|k zwKdAaC^J$+?}NGF8&PbFIMP(%6GGACakqBduzo>B@v1M0YrULY_!ZeR#u{1U3mn(b zC|ojlM_lZ{2um~PcG=y%%a3lz!?GS5nG+n3%UV&|LN`(I3nHj%WH~+3UI;5a>L7w@Vz7wTUvji)3_)WEgKdyq<` z{~)mOc}Pyeu0Eaou$U<-lqQq?g7UHjpP0=Km_C!{n7zeR_7J)iw2wLvp@EOq$~~GU zcb>>JdGr}cKNqWhgTEJ%7ilo7&WY5-?g8@!;8A~ux9f(yKj#~SZqy-~Mz^0v<42yA zKic4pdPLn<96L4*DU0rwx1p(iD4Ai?8W!0cI8F`JeW$?HpBkS{NgW0|OavRWBBSNH zgeDXfvD(&d8-XcaE#~<2wv;6*H}p&S9NVHP0HO=VYcMQp_3b;CeO+6lO`8L`AboIn z2-i@)!)gSt|Hc<+w%)#?)GlD0d**^7bG~4JR7R`cp-{IyTx~k?T2@KMW>4me-{ul{ z_x=v)gMrz&`v6h8ep*U0=H?TRgtAMK90hl_>k4V2=^7dOS|j8BTeUBQ7sQon_I2vn zC#lIzV22YQm*rjiOfcEctJ_H6w$>{H1c@Yllo?zM-FlKqnPuk01foJQ3eMI2pEG< zwd7;`cJ@vgXZ+(mZ!1v~Vh-{eYSb9aAq%Ceoa(<@ad`HE$LH_oJL4&*`*8}tfFL5> zq++V-2gS7Xk~XS+>39!wOe6&kMVIEZbEto&!bFxMO4RbYlLN_vU4>@g3kg@7b&Bn) ztt2z7=4j@=Suk5m+^maCRbK#iGSM_{X_TQ6hLGh93Uep7gpKcHdf;$FY{;WYH2A`b z-#~H8;RWGcL!dk0)i)w>Orep|snBX=mPgtvCso}^GyACI44U-EU~BIQy~`Fm0bk}2 z`T;Yg*1tmz59gsl-{jY^k@4Gc7*FKn&ROzWm%?vAl$k+)k1jM%siVG%OL;)&~w^HgIa=rE~wUqAOYU{=<2k6RDbxEweuX}>=IhBO;9LAZYFxuNT{q_|2 z0J~#@liD3(?Q)$APlVTzpc*0AYgOzD;NK2{Nc;`+0FX#2_l2)-dA0rKR~}jQTw<8( zfqg!AyFs~8Q@D&$rV7v%Zbbhb;b6FO8TG7+^Qo@eZet=@_-%88ghSBc`L^wEni`=y zl;kJUCE|CzxnL#0k9Yxl@mPpYI|&gE<;AgGe-~W^OZSI%0`ODr_3OA<BE>tajTZz*Vfz5J=EhIL@ zsVXpD09#JA8j?)l?}P4@q_pim_)J4#j^exl!AS;hd%SkA@c4{`N4x!@`S>BvHy5-2 zY*+WWeOWwVv}ba{{cWqSjD-~%4R~Mk_JDS)ljhV4bfWZsda(x<;S+|7(Ufyr5z7#S zCIn$w5mL~BzC5S<(#{PxZLJY#FTkvKvI+-%;ulit^IeD%K z_fWi~5lQ%Lc|ZpY#WVW*q-+vfUl&}D84 zUEJ{b#7#xp?w|==o3nq0ZLd6nEhIN*MP0h*X}K`)skK_I4XMEAl1L&Um{NJ>lS!mT z4e6FGr01oj*-n(mh8yO>H)oA4(GX#|o6^S8vBg{aG+=mQ2Iy_Y2CrdQ6Ur(8icxUCQ*tHPT2KT`ER`ikJsB>1thLWZ4czRrX_T+<53*X0Tsu`uYby^U`Q7P~Cs7m7zaR8U88_?|MH~=P|M*VHu$rN1REjJal(c zGto8%1Z?Z{i|UD3QTRSLuD?>d305|={@STQ5z@2z$*QCjLExHFf~%%wUXRl)C^>NQ z(t{LRwy2e+PH=GW?#_5A9j5Ie6}si2X_;R;;l()hnjoaFLPz%{}514VKcw%g?OW0$5+` zi7E{xwOUhNQ4;$IsyPsCAL|IEdY6OxEGRXsB)Uziq|LB2Y%eKwPY`}*ek~XW1`O^D zXR9|M6~2hT3>bHQ_uxT=B&vlK-EGT}wpRC#9?4@DijI9=z?ESwre-B2b*JGv>aaSi z7KL@KrqN9zWz!xjRsm4(EuVsMktzvSzk`Lfx2`T9t*y+^Gf8{E9bz3L10?1$3~1ky zO^Ts0j6KGREI+0U95TTM1v?HZD)o|U(<&0DS7;>$Qslk}Vnmwr1IrNsyAQ~|dx6@C zOcvmZTy?tu=~G87A|Nl*)N^eh%+W+E`g6m`tJ}4d+B#THx4O{;yHi-FI54ufsO-6) z!R0p<{nXeTDc+vjjzRtr<+}uuU%s=(yFz_%fOYS=>;T&!B$5cKd8; zq6H{m$8dr46{L1suSJ*(y{KE>0e_z2? zGIa28b8?ik8zss+5&NaV)QQXR4Iy+%_2vg0j+4T#!woF#qyUdRv?WxY@-lh0Lghh4 zax$75iQvU|o)ZruR)6yhCR-*n-kS^cAVO?km(t(YwucKbK9T@R-;mW7QxdRvZxP#E zaJntmnk3r?ov8;LAndK>0ridq`2we?4{@yF&aFZbn4MgUJgKeybvIsLJRMFQ{t~K@gFazjm7YkeuLyxWG19w+B3RLR_HdmP#pqjTuxFkAgpo*x~4S^z!N8Si2(5) zEq@Ia{B{Y#E~)@0!0iIaHj0P`>~aZu4NVvTxp1y14E0-j4U-rk8xJ<}^hZdRR5-)v zU7QU_8MF?q*nA|AvAHm{sH8<7+k;vUCJJ1q!+yb#t!>J~AalNeC7-=uD!AW-09551 z0Iq@)>Wv9o!ghVXyF##wsj)y+Kn59zM!F^zNalGbiOe<#B=tQfiG0-ApZH-IPx8mF zrJA^f%?J%^fj=N&;AHQf=+8aoug6_+L!aa`o0v@!)0v_ThBsUFFcuamVjfix3$zH< zwGgVfqA!-uyvA7(a=rL{g7{%sVnC)HozA>POmqX2cmt9+{aWD}@aq8TwFrXv;ua#? z{8=PEq!b#wBpV7KI;0?i4=(P_b{$c)X(L!kf(P@v{L--GJJUY0k7qy0|Gvsr#1{{fh3 zi~4+~%H2N&`B0D(hXG$#gnQEQ=dxbWjA7XvaU{rR2 z??b}RVr{zB@KiWultNCtGE_K4lw5^Sd#n(7lw4jrsI5z|)2KW@KwGQ`X_ReOS?2kS zWV46;_glPYW=nEtVSDY6U-BB$IN;rf0j?r8SH$(a#T2$jzW9*MsQuamZ~#CnodQ?i zy=R{IIEr(4_gd5k##97E*ss5xWeVFo`ZS&9RC0{_KIdc*k$`oFy1;Y0xiSthaM~uY zB~dS8&6Mxb&We>^yrc9^sCHw>J#N{E=q2%Py%G&nz5{cMq=~QeyrSBf#=0r)5b|j3qAq*&tSR#UEBuyHJ&>&wT;Br&=x%&+;~d*jf$VVgE+z zH>90$`wS*YEBPKnjyz76Sm}sgrL3rwH0t*<0q2?VdgQDf?kc;K{dtQkxs97vBefKp z@mnN8@opm4e^3e7J9Zpl*Z>ZiH0a>Uc6z z@Ovl_Z#si=)+#31+*T@E-0xI8vu=Uf+@ZNIdEVV$mQVs;yt0?3XPuTm$tH;U>9^?G{#68Uztk1+lko#|o=kwC< zx-G3*55v;K;xOl{r`ij!&2TapdXlLJXEf!GhF9=HpXY1hge2SdtYSOM^F}`~5G88p zt57s-znd9>QTsKtC`(X@m04GTqo$d6zJ`_7Ei-uw>52R%L)Q)XkF6YlIlByiiYHbiCABy?2a0EIvh@9 z%Anz6{rkJF>Sh4?TV&2%&nNt?hln#sUT?+)&A-9|1W%7i@sPbj&Rni; z9}=0YsNotQ^jU-EX{us58YhAm*Oa=WFXfbyFX!68pMG_w6MtgDtpxrHgkOhw<0Pv0 zN-U5b4_S0z!>dl!_F?kW9`#eJu1FWx$*%9aL2LPem#F0RUh&b$@+m*GQdLqkb@|h^ z#eE!KkS+mU7zZnakM8ra4v{B7b;efULQ^D@rBjur$zMLnO|vrdy)9!c$QGU3^&qnl*h)e59sSh2EED+-59Wjoqc4;n5lOd^fg@@K}*^m=nX+Rd%rz z=F`}2tpQ3FZOYKGuo5dA-1m%quxv4_VjU|zk9qup=(goh+#SnFl`tlpdXOtOJIHW} zkD>9^n*i@TVtzx{7Pa-&9kk2%?E%MI6<0aIZa{W*3o{zyGE$Ros0Iy%clLZJB2aGW5gYD3vJ0t*gbGn^zge>#x##!Jg zGado!CE}si{#3-|vk(hCIyfE6lz3h1$*DRS7E6=^>3G&cILP3N{kt`=tX-DB+ ztf6Z`t-qE;Dz$>eZurbtm4ZW-MW8Hly!`xKXT~>{9y-`}7|a)Cm64biJN-7_mYp)a zJ>Z5B40(uPy~4=v3RNZG~!xJdYEEF zO0|8_p3zuz8or_Zn_zo`T42_$G!-O1iK-?5?S&T+j+Xf+@$PDsiWAAPK_V(oEMI@=WjNqnn=XQ5#Nvl=WQqNGIp z6rvx3Ub*%W5!igOKIN5UQ^RaHRQ+mg`QTdX^YoGNm&qSyRJVSUAv+pS?@;*rW;*-i zlD*zMl|x9+Bv|t?&7vCq(;C7U-q+xLopp__!Aj)-A@XJ_hiAf4W7*h(o%06E)OVv? zL#>t*>XfT-4@753&d4J1qeOX|Uw@oL{voJ%XtD72>1LHH^o#kb|GVD&DoTRavf<9L zDQ>X8yrMgMb(av8iFEdu(?uY5E(r^X5mBPKLQ+%>baTulNdj0*N_2l1pp#2&Ky^a! z)SuQjSnNrzyVX!S*Bj>?BY|KF+6b6}r$CCU3x!^~QV)MwoQ*HZoz3&?G|Q%#T_!;` zypGgMdOmS9V1OEY_}xI`%w%lqo6$5*vC+CFzX-q;gUJoH+jdV9y%Vd+^dp3pM3epn z2s!BXi4sK3UHYrBW#eL~1aIu;d!28@Yg6H!0JQT;q3M~y1~Aao7s8gYZWd+c|MO zYqq=i{1aFrpxFab6coYQemQR#9lu27rS#uy8+z_XG7`iYd6_4`?G%GY)tEqOWR6#o z3bTp<;O3nja=+_Wr@XZ2Do$r((YfbZYtQG?Jfz_Dn}2^#Ij0qw_+2>#G5@yO2gB)~ zxQPVkIiEQ#&AiX3wpy6M#nx+U_v@7g^8rACVHJwP8II8_5E`a%>>dcseifA(fAhQj z{o`<3QHJlox&6VV+f|bqFVe0vdqrEk`+ zet)QQKm&-f!TRmSjm;x!HXOB%hzTH1zcu>t2R8wy@X<)*ombVRQ>GN^o0nY%Y|B|X znn2)K#GJ=0{Avg&ia;0rAM}BD$;-(-Errkm0`78pO}u^dvxC7=tJ@4i)w2J389$sh4soF zY-dETOXnMHch-)BlehYii0OE{E|$NVtc_)6CFmSxFNx@Eqoey0H;=|klRi1pnTOUU z2+P(uy7;UNMGwRmMiK@ZL!^gQ1}meF<}IzqFtwFwl`$L0j(zJ%=N;_)p{K`-^OZrD7iMXWBxcm~u;^jOC^qVt( z3x`iHjfa;4%j^y@2++D4Ci8W85bw)t&cLk!d|#>uJ;In8q&oDb4SSXoU@2@%s6<_Y z-;f)(EeLwM8DVy;_JVa~Cejo_Ek*Zr2Ke9^R=vOyPx$ql!o~?%$*?4q za8CXM^!CDoLf)u4uEyoMmIHq|5PLGC@H9Nuu^o#3?Q9LmIFc`Bhed7q-*l(Gj75Hr z4yCm_D?D>0B?~X5ghzdNxZfh_v+2^NN>vk_ydT$4= zHDlF&Cw`v*OczrO>OeZEtig94+Epnq*#N$_AD)ZTGXT4j6aN*AkP$EucQhr+ zEwm^0uSE0(v#VtxZA6GYnj~f@h=QQ5;{$SmU8+VW2z^TZX}^@9M6xQC@u(l;MvIVT zUG@V$aVcQ8HS|aDjtwHPZ!_j55GC=vTtQy{4;^cwGt{D}T-ThKpO;1McW*|q?xcT5 zw|BedC%yWKy~(|(PU5u@zxv;V5AH0B**0ky8`J;HlLkb4*SPxA({W(>ZPf-ssv4r$z8`v!cyJq z;mKu3qeJfi;1${JaJ_(A6{e}&5UTL92r2`^F!YjV2MHaNs;%}l(~LL;n^wQ1x;o@N zWlM(TSDhbp?LzyIbpz0p8qYZW3!g!M;=&mUxB7+zYn?lDGZJVnD!0b-5+ ztrYEjzg&vZj|)U0RoOyT@t)_AcgcTKFFFVP1F1pFJql$bPo=#C%OoUgG;@ROIV9IW`P?-C{caLlOH6 zKVi(Cn1p?OU&w{CROi%R;Ik&z=U`CE*a!V__7uAZq2&z9JncfD9gs0OTX% zVL{`T2eLX#=8neePJu72E9wcWJL{X;5xzezQg&t3eiW3xNt=03#-IcnHO{{K=<%er zlk6R~iBP`U`9phG{+Z0R`a8IfaYsFR5wm2aIg{|Gb}yrMoOvh_8wlYTdkOrvLRSDj(N703d?bkOooF3G9uM6Sb2N3ARcXF<233Vth9T;oSc(q@ z4UuSQ*|EuCbDBEd=&XUK{nm))1+g0uf%^R*e-QhkdH=P+QZ@VFdA}(Nqscxgo?Be{ zb7VyBP5rzz65TT3zi?V;h@Hb4RJcPa6fv`1ZjB7al?zbdDHA16Xy!|)b!oE!25LL+ zYvIH$aZbLkArMBBHHMDk2Y%OoarChsEvn>}mDY7 z^gbBU7>?YV%^7KN^5_h|`lG4F=6y9Fj5cO-S$0`&_Px(L_Nh2y`mWm6FYJYUV#QxlvMe*1nN}?a^|O?<{>=e$(=yk8 z9vuLEV8(%I^IxF?9`LhJ+4U8_gQDXx20k=~u346of~8fUJqJt@xc!B14S;+DSM>3y zagayY`lJ7nH<&hG0)4|WZ%e}NSg|F!RG2#oB%Uh8S7H)9pL})x^!(|Lg5L|&SNyNw|7xqe{qN}Ve zebA@y4D9G%oz`}J^40mX^QR|z&CgEt>OIT+v)R*A5~v^9fM{06RKqrZDNLEZds(C= zhTI%tFW3aS6}a-2+B=@wHx|C22^XsuQY`suApZUM)qh;QdVl%$)w`REH_yq9|EeZKOTMZK zQ{foX{Jj|SF23f~X<#37R&XfY>%?a;*lg(5)_F8!xXKOo7c676Gpqwh#iC#oT3p@J zu2kx9jBAo{Cd!-MN;-{=c>xbQ2M_z0%fck86*OULPOhumRL~8k;b}?t!@4Z<4@4H^ z!>Y)W58VPmf28~jT6gvQNki<`2Ew?V$c+unirJ6)X$;`fWYz6}2Ma#i8-RARW=Ppo zbN4jLd|3U+^F8)PzqZc?Fp=hxi=JH1W~Qfk6a$5fM;;LYGHb3~@w|W;k6IH>;`-X{CxqeQb?z6x(qbci4BN-F1I4G#L^E^%H6~rPMB@S*9 zRd!OtZ3BRzuijt1(D&v~H#b+da77Pt4aD1Vp;%HCF?auRNm8yry|-Kqhyj>;D#ajv zPQ%g^Qm~}Ib{x)WA!Wuog*K(-ATCy~3kpzrj;g<4YfPEeh~lh`dO$u~?^F@)u< zi6*jjAPu9jjjQh|hYU$$W1wYkts8uAa6_N`-R{Hbw%(J|d+*<=_wTj$?~V6w<^Qfu ze}CG&bwjy+nd%c#b~ftf%{4J%Nh#qHf;Cl{@9hbF(g4$b}k4SKOVc)Sq(@-9hmelTVTuUEQ zQRAn}Ju>mdbqMc+roV{sWps|;dF5|Rr`>XNA69wCiaXBk&+lcSCEl54hco1E>Kxiy zoc}VWD~KL9-@N?hIk`wvqp($qEvpPwBQ+;9=igz0lQVM5hufFyQStQm4_eDe<|vJf z-0f?~$A#V#uryOC&7ig{2p?=!M-cB^Y(rd&kzwXukB3TsIkZbrAX*oPVoAYzi=W_V z$qM>~0TS~05D9ZDahYD!zZOj9D6d2y3rI9rCdfd#WxW+8>M@a>e{8a2?gG-B!Mlv( zXHRn?z$f%I54GED);y-=+D0|JxBw~3`VYP02!>t-1+=j=ECT8DY+@x%R5?8<5qy-G zhK#~yDxnJtJUG+RDEe5#mCPrBa%dfsKshwKpB-hN1j`VgX=TCED1WHzv zdy14vEtZib&)BYP8J4P0>U>b4(mFP-Z+ZS5*dHU;^%8v?HsjP6HGvilNRly_bIZ1e zZ8X-3_$>DMmTj*GxY+sr^aP&_#II6gPX;~t5>?ZMk3sqHXN`<(gNSY8&vsUr6+ zjzf*-PQZJHu_1;t#c`!!juhrbQJiQWE`-jXa0&j#{(fu1OwfM>+iOE)ACQIx*W16D zGZ!hb^QQnrQ!;%Z{^-acIh3!j?K4>GKV^9)x3-9kD*;rg&dUV>=lUWS8_b(6C0Q|1 z*PHd=UzfDh*X^2>|Srcl=yCIiLtZQ4B;%*ctqDoa8W((n!OqHa{Rar3F z)%WDCbLsSs@(lR0EsPCK6m3Rt{Zi)^c$2xO%7#;=9VrC?Y{5eZ%mlIAl#(fzbu)IJsO;d5H{ zii$1=Aq^*E584P@wDhuvOuxN>#@Eo}2MVAXVFIp~0ZERZ_jY6*q;y%((PkMZY%Je@y%viCNJWU!V%$rUQ6wjM4cRv@!DzV|E&M$>OZ7MJw& z)>PcpFU@Rhd|6cN41c;Wc*!Elrk^ZnrkIw0D&ZKyX|oPKJQ7}3NXSvuV()utE+8Ob zeJvdd9{0riy@tI}5`k2QRXoj}znh)eLYwlgAw-3(N~JdPl+(`CljBUA+IxyNkE4PBlg8 zg~{w{>pm#{%)QUF52zn?TL)+7txo`GlU;|VbepQIXqIgW`v_@_cl+Mcfna*eg*yML z=}ecvvtNWfE`Lb2L825Z)mWe%vow{{WmV=?*^oU!7c$Lq?dR$ibw=*#fVe9OnXeDa zG`77!Y2=Vn(M zh<(hZ3Mb6ky%}rSP>2`j29thC?-j%4F7$HspIM*%A;tu?RQ;(JUE7V%+q`7Z!Jsn- z+-qyciD;?_JH3?^xu+Pw50qw3Zg~Ejyb`Ly>ON)bGlTXC$WL=w42xQ8w+%PN%Jvc= zuQ1_lz!7!CK9*k}ospxDOQruK%B4D*lgmw>@r0LJQV~m56o@SgW?l%wu}-SU(bJg1q%lcyk|6pcA;nZ&+iLfvLH2zVnG_k zhdTE*7Gz(ri}8J25ICbBFuW+tEHb$u2H78C72m$U9JPpV-(T7_=zq%P&h%l5VM6&Z z(;q3Q)^h6WmW*1na+3%boITx>1SzlU{fN?C@rmx7uoE^5qgRhHpimjz>IzNP8>(@PPx;iR$D0hwY@ zT?2>@%?8M)LCP`QCLD0v-s}3E=5Aj%ejm~va}&9H%eDmi6fR|@7cXk@oL;%cKdRe#H7`1M(y z*8BhYIF6e}xv+tVJ(^L@dg4Zz=7uJVz+zm!2cH}H1k&GUsrN_m%>%YGPR83`4yoxf zMD#1$nS4+bLu%!xGv&0P8&$+;p9>5t%k*Sjdu4XzvGNQdqZLW1aUB+W%pi z0AH<3)A&PUZ^n}n%fTY)lmpFih>iNgIXE#87JNoBe#^*G{jrc$UiXp50NEf+(^M2E zQRAXw6e}xpOb~{t=ggGCp0+`l z8(Q3I9nnzaIb@wES9GFa87=u8dY3kI>t}Nf&h>acKSmNF3!+L|m^-h356J)#2K!dY z-CUy>vUXePi8gXcsM)EXuD;A|vm+Cf{sP>5(5f(-0k+&RvEz;5%HQnCm2q_N<`T2k zGUhX5pW#M;eNdK+;}IA$jxp>R^NwTowMpd6h4%$2OY0z2FaqKjjyxRZjN^g%GLAjD zGmcmujU$dz;~4Vr;b`13>*mAd80?n#zFr>|STT_vm7O@E{fRM0{B`fqw7T%Ssr;V$`B@U^m&cZU+_y!4@Knl0ADvZy|R)0xkp>^)Q zMA&*j)sK}{K?<*dFt+77I81lH(CFIb(jSc+T$3rgVtebLYy&Rtx_eMgm9Zv*t9+=t z3=Z$uP{ji2$W?u4q+P|P21-KnJPSJCqxM?JlwDPejH~taXrq7E%NTs+RQ92!xWsuI zI&A-n3B^G!X;s|O$1t=%?F%Czr!QlsPLKZ6yL$fL4X1c&Sy8|7t;qd%NJ6trwd-M6 zvLBYwHAqll7Z%gFMnbzI5W0a<#T?rav++(8MK%(XPX&>}0yrZy%j|dRdI90t7R+3L z5ZX=7mM`AfvVRn{)eMma*wQCMrLXE?8IfgJ`-kI=$s@cd%YrYeGK{igdYSoi7(C0y zR<>sZa*bBqfsD<}1}EM%i?C3;i@Qr0YoP$-0sJvgw*@9FlL|U60zGav=|j`P@W}*G zHzGj-=DDv}o637}PmA>8syBbEDU#LC`$DPxndsl?0@~tZ4nx4vRGXY?Uxrid5}j(AlV)5&QJn_8v1O>jR=Tl% zi`lh!D?LfaEvv=yd1w#1gbk(@igY?b_J%os%UAt z;BP4W^R?c$o(fxCgDawixsqWq6IN^#k;}72T;9>$(*fPxjuCDtVi6`({H{56OW`?F zxki?{aCJl~d_8IhEL>~c*+*DzoPnBvc(sf^#ppu&fNRa}LHhk0RWMo@5c_oQ-iGdh$gSnTV1= z8d63C*e}|B=rPSJ<4N3gB}Qz5Z{LP}#^Ml0+@II=di7N^p2V2DS~T2;$H(*eF$R9r z3PV*cXa7o=NMsu8rPgySR0(2LZHn@pryD!pQEo|`--EV@v2}`G?2>9{)JZnFn4WmQ0Kr}@|AcRbr-fs&C=hQI&q6;;Y3u3~f zA0LS>0>FP{Z?udPbVdzv$#~uv%@JD(e|DNSQl*rjXPR;{OOnZiW)bD`NK=Xdq8JQMdqrP5pgHCb)JxS(}0&qtvRGssd%htN`WTAVj$Y_r4Q1m8*!pa~TEC;%}oZ zVC@c}5+)GslU$$AQr6j1!Wm|6HsURcyB)d5o9xZ24DAKQ^)6?zCDg`oIqGj6#W&7a zW7uA@O-$t$9#t(Kr-2v)i6Arm|E#gcdiWM>i2*D9w?@MP{o16%q%314WUw$-wjZNs zHu6p*=MRE>tI*Nf@19vcC-k(D9jZ`$6^x~rACoZ$zNmycBa2fTBG?aDE{V_zg6rOZ zR-}UI4Td^4h1?J&Xs*^0dfg>7rwK0uP2~!j-0D}sX9v?eBgv@=Z}yL^nc`Ef3t6rD zgv+()gF_ zu(*uY$f6l5CSCg|fwjOqqCDhtHwF0h4s=fP)S*@9pZn3FfH*AMTY!mFdT|{WSC`7y z=JC#$0@cCAXHu;EdRz4k)$BkJF0>w=Ry0FKfRpb9`QrK#!SdC*oZU0<>>^LEX^ovE zZwILM-!cV^rZjdww8rZt-qS7A%`@?kjS!Oo#oAzG7Y4vGFW|x^RDduEMsa<#s=N6b zj@>nCMDPlQyPuIID!RcgdL510aKy;(cQyXpuuY!PGCr2$=zGsXoSh6Zy*CQ>cjk==*5rAzLEKTd)GiXY=7p$XN1q$-2Sg1 zNDIcYfl{t8#@?lT0K&r(JbQBxrrswD_rq`X!u6ZKoBh`7cJR9on&TAufru6%<_$lt z+cj_da3$00oF#M9pXUQA#)OHo7^tvyB~!1HGS0bRf3?vQaAeMIo6wSp!GL9(4J%}6 zn(8jWJ*D^L1gwQv{NRj~QUcupS{Q_RAqe}(6;wjOy@jKp7;NqY6#Yq!3AXS}7*VA` zpq`g4I%`ILlSx?x$SGENq_)@{lFHOQ(kPzhO3g^vN^fd0*4>EfF~&k)aYj^ zsYJ=c(MS4C_kGv~Ur4!J5>+LdDdq*6S#f9x`MU_JDH@bZ6UDB-w5Ax@m8XM?hqIx= zT*-DUH>_QXq!xwXUiWb{5=xN4F8xxHYwc)wy6WCCJ z-NOo&NTJH2O5mg@Srqb?3749nxxr0H2qZQt08hq7utMmDB*~hJ6{}OZH#7wt`Loji zhO8O(H0V-G*t~>eS!jX8dAxoQuBPePaJOT-g*@wJJY4TE!rJ%Ux@65^=cNY7M9B+g>l$bTHyd3EMC|644;!X6&0eum z)xnv_&op5)(u=X&0JDQ*7=%iVv7%csz(Z8h@FmTChZaA^MTh<1-pVx2pVw;LR=->) zUtn^N0`nF=5ao8*0F<)b+8&!f1Ii! z*nJdCE1yw?)g*0P0+_vQRHFT6oa3;b7dA)+7#CuToB7g18Z`JsL0#!(X_ZINMWz|L zhkS0nO}-bmr_zJ;o-K`~#{a1aKMHEidmHeP`8I&hT_9@|U?bUk^&(9=TOAMU+o|G0 z7|NYl`IT@wR}vA?F*c?HB55HWEbi$RHM+REgqN*exPt#-xqY;g*p3C$#WDjxLI9n0-Rp-I_pz@tX0P3Xxhv80yH&?U*3F9Yq1G8Vap+e2Jcf+PV-&QY|8Z zWp0aHbZtA&f}(7y3BHzb5seF)+}_h7RZ!r7mK-~_ouSm`3s~a#IPQwp$IL}r&h!_3 zQvWUPnzm)sQz!c>=6ZnDn-B-ID&>ZjJOK+JUwYoPjq6fjV=3aUD2O5;ESbYHcd}br zjl}!*MtazVN8{~5+$p(z!cvGv9y1fZ!4MMgq!%pma&ELzN1+Yw($<7my9I|msj zCH#tf4?c7c5}H@DOx`O}-b-ioLg_E=xR!<*sk4?kVBl);QXge9)JNPw1aN2iuHUmP z)Bg=o>;cnvVD)?@FuR{CYnn94H798y^E_5i(zt%wUOj|?6n9KPFpy393AJbN$BGG7 z&=MSfg|(qxg_sdBJWs%2=MA-PKDx9yrw*h^$1y1shvX5s;(KIFwzU@2)a4$ zjoAQG`%A+#fuVW9?zpVt_Q9kvR3%Cnsx-R#ny*}ER~xDfp*KYuuMJ~JF)b_AthJIr zK*IyNxZT-5hQx`<7_sbXd+%YBZ%Cs^uKnH^nOtsoSuz{u>~03cr=*ef_55LI0yt1Gu9~II4=rj7!Z$g#LV&wHGdXzHaWHxZ5{T3!>|?@MY9{~Xk(*JEV+nw` zniLIuwlA8>e00Gw^0$f=TcYJcn-vlp!;?s%cxrN2jJGaE6BsL|fSc~sM=cvewX41K z>o$;PaYERNXq0o?=u4~^EZUG^W8G?cT)cZ3uSbmtVqu&R|5YRz-oz8CJf!3Odj&G;BB z@>ht;?xi>*?_{a}^9nLZM&hL&FC|m&WC`zol0(AfA8bbaB zN_Y5NkxM~jVRmn*(xH?syQ~SSV48TB&z*YJx1~<9DCyRxbS;B#+Vmz3^%}PK5daEUP2zLq>f)X z2tscl(dpA@wtJlEuH=RjMyX>%b4_*rTMHL(f&OQN9?J_gCl^Ffswb(luMR4y?@w{sIlNHA+?X>G&yN|IMaE|ux~)V?!u zl#{!P#x^|^vzFZjffQ1cwj2fVqFJNOIGCXDl<_Ug?v03oZbL~?ZF9YDaH^PCete8^ zH^!zOi>X?bD*YI+QZ^R?>F~pyt?f_@n{a#m(j6}uS4_qqP>*XND`D{7R{1)7NA>D$ zO#wGOa~tZ!z_>{+^*))30ai2c{Fh5s6nJ)vZ8znL`b;Vro-};v49~7RhI34IoQcq- z} zL_3ab)-M_(_fZf^^qHA-zp6p*ol_sT2Z0X>yV@;a`Kqz932 zkZ5AkfGi0YNmiv8KZGPy)2bLczsHvDC5_atS{cyhySUJzSOVWgcpVLMrb4jHE5wYhvEDh;vZToxc>yJw zZe^@IR%9(zNj5ZDbKy#43Rg~hAb%p4y`oBt*?MGa+GEq3)}UwI+!e2V-Lm%WP?;BK zr`CSI?z(5l0BjAav%s9-*)Y=fH_IiL8|Sm+!sE-#yby%T8`90(G|ODedy-0W&ma&< z%c+oApkg}ac&mq`yhLvDLTVY z$2IlD?f?G!ck#{Ej_Y^&PV-(CS$faY;UaH}oM=F&p_N3Kc1LxB4I3}-?|C|6GvM*& zCRo*1ZvxS~KzPbTsV{dCmeWoE+9`R(u_PbBhW0F*1s8RF$NB(S1Swp{KB^C{X zq7SlM6U8@GR#L%arLt`hoj&Cu3^Uh^eZ-3p$7J2(O=b&eTRgD`HmEk(0F^r9Ha&No z*A>lUA6a6Gt>js?;=QG@K50Mvy0v$upvZUlD7{Yjhq8$MTGW4;V!%yjdwP{^25z!LUHIF7TZ>}l+elBmEr_q zY;Kln7P>$KC0`N|4qR-`u93}a?Q@+A2w(jT)^Mi+((w_%86+a(zg0(9a$HS#vSy{h zHUW`$J|?HGi5c%a=411zMqAnB9V_lQyFWJ>U$c8&u4g7$`5Xku^IyV$2CWUKR9&R0 z$=Squsir!^+-vcA2B|$~q~huChpb|wM4mz8#Xj!B$8P_=go3?sZG+6SEsKWv(Fj?d zrJ~=O(PP0dch+!ITguLUz!=v9)UCMVHj$GQl@R@h7A2a|k}6gXnd6%BS^bWt9{wFM ze**s#O9>qy8K(9unf=a)d7=SjDGRqTw%t&e{@Up-u$?FB)-XYQ+scaEn*!rT zPr2dwbMi{4s$g|fpw?%Ga5z$HA+NC3yn1f+lOW)cCd{^(G6?&ju_5@eD0^w!COV{L zJdV9Ir$q_sRy=aLS1c*?eh`OZ=jGBS^fdk8T)BtBO17&Zr@4@ifv;U(7IL1lyK|M& zr)MyQm~7qVI;kQ@Pmkv0ns0bUiwp~bxvFUPxv)uG7H%RC)b`|voX7%Zlu$u3W($iM z3bRG6U_fX4RAi4lxcNIzM~N=S_`Jtbee=@Q%i38rpqBjc#j82_)~GUuEhAOa>(@99 zPkhRajk>xASO{YD`7jS?>H51qho54@!uEt=%>*%OZ<&gh?^)g(ZrGBf@ZpyImzVkn zsW`MUr6r+AOp+?2A-@q?f9K@otE=~~UR>O~dik9E(R_jX!tSP@-9c*euVqnxD^)=a7?eJU_OASZ0GI zgFp>$swd<3Z)zgmoRDv9a%)&(<2&)1F3hrSVcGG8A=(0Vt7_p-)`V@Y_?@Zcc(5(H zo=)@;dzaIyf$!fOJ`cQ&BW7V*e3>~Uaxv~xG9^Y^bVUcP-b-=v3-C??We zauF`k(CVXYU$MAjfABhanfgm%naS;ADRY_0Rd)~O5gYMK3`U;Oncm^U)tYiWMx!a$ zc0=lwIcdIJM`siKuM??IuhwcSaD9!Ab%S|wRJ zPl?)YGA?fGCPqtXiX<|k|6uXAU^JXx(B=*YVZg!EzZ_*{27|J`XMeI}!UTKx%KXt0 ztZqPJ|A|)~q1qonbGb?sk0ZG7XoK1ca(VsY`m*U4*)x7RB7i!mAoux2`j{!K=3!*j zWKHvns$9S13W_ks3n)8pdN~-_DcE=;2-h1OMARC_i$7gl5nAcpD@~im10{ocjn|mt zJbH!Zy#v-~Aw0(F9yWZw;l~=Vywh?IKUNr~nb~MRR@i^_2}1D?XNmqKeLLxr20@y};OuzNEn zZr>-rHD*QV_L2SWd35{ox>cd=_-brLi4=k*Lrrtzj^uCgWRL$%v@E5#)c;%247cyy zLzwxR+aEu>#%p$>^)bA2_L52sM0-(TR#p~ND1B~Cf61|NavqbDGRcGfy$8>M$tvNC zD>%;^JdQUm2PQP72X+DiR>xbk%=7Mi{P|zO-^d%TAT(#FA3Z$Gm1zqLlG~M5I(hyV znx@5b*(gi6+ltsUw{1sB)PAi{rXAp@IP(d$!JT` zC-p8KEY6INEpEgRLv0EzWnE)E63U4l_OQ-{Gq@ioSU?EC}^s=-Y!R`uuNwQ=MsM*ZVhrllL_ggf3q5!ug>pCanql z!k8rV^4;}c-dz0Q)f?D1jBW{7B+Ipt%J+$TjK8&g_iOOjM6As}&(U~C)UaT2 zaByAJjt&bh#h47tV|?DqEc*T+@hLR<*My%b6;peyjdedMW*ubkotbs)OXl;gQbchS zuJct>LdaH8s_9?gTA2(sThOShHj<5_p&PH8RzH3}T)m-onJFm0auD$Z2;-td-7D?n zl$G9SFxWo?gD(`(nzXyqCq8A|1_FS`jqnZ|x+~er@FQ zx_j4f_gek-`tD#@B6dpa$SWB3gE+ri>k58eS8snVIZ&0svCZIW_NvNou>OH4KZ54i z7mOi8A6!jwC>X80yY$#p#Li}KN>a$bLaER=B4VFL<2^R1$})W_QuuAKOksuoaC>ag zw4ph+fhVcF7x%PC1I?nN(i{*Ghst6$6CLVzAh-^Ey83@Ccxh@sn+nmMt{rxjh}tQs z$%rNy<}03EOc0rrG_$QOS$WS`SektDR+Bdos<7} z@ir9FPfWX9g%zuCMsOfLV0A|R{ra1Cb)8i6*C1e1iC*J~+s-+8>4o3}2Rkf5 zx#Z>2D{ae_r3|*<2w$yj19FXH86T)559SoK3)`AE7-k9IahwE}a7HqI%g9ne>+x_d ziLIerFr`I)Q@t8LRAZN7!6HNt>x_)(Y1{M=nagaO8JX@{Hjo@?!N6sRaPl?2dF7zu zF_;1}HQ}L|6e~Ej0sn~fgN<0~vUs=ofX61jXHlrtzpZ-cGpw3 z7*`WqBVa8C&k1>@WC<1hG-Pu?$`uBOV3OG)lS#Oo0FFd%OivM!5_pNM8BJ~XX)PAz zIZNk+ywI0%$rD;eH8a$2r!1|sTy1hh*Nn&_Wmq16NfHmB;X#qt3pfGSGhFbppv4x2 z7khS^SYdyH$Q_1RVYf!ez6y)`WOPG2z_aTS{UuxU310;fZ})PkF-`5DVGc6vHu4kn zk5EMz@Z3m<++IBwr^d>;*5JyDX4!VeK5FvJ?^q18ume+CO+N%yo*`1_1uL;`&k?Re zNXW=If=44kL*s&~#Ht5}Z%58OdU8HeSI!6O%XvU^y#0D}K3aFqN9xb{ z7#%tf>Ct(>f$h_$^YJ=$K31>J$LQAii2XVrrDNx#^z1yKYv-PQI}gD1{d#xq)4g+S z|IWQSc<$B1bDurfqmSpIojmtE`1n24T)U{) z8A+u$E{%C7mnjy1s`7Xh-wWZ_%yjoKPSmMJUvV5uo#`Fr8C_&n&xcSawDCdmu??>3 z`dI9|m8;MoB+weDPvL*CKJ!>Y<0jW>e|%@mk4?N7Ki^UsQ_YX8kX=TfF{n1=d#%S% zIB`81c=jGNk2~W&f>umk)viLy=(gJu45aA%Tfj;mt2l-o?k8~-y_xwH%gf3)N z+U!{Td&G7@1@ux{@7n_#r$?wJ%~VzhJ!rw$L)5g&bZRky=7aqyNXzT zV%J)yj_d1TLOm+sePhgiLSHRN;aAKQSEkmk=OwAHc@j&pI5y@r5G^LS!#;~G>QE2G z$|}d{LWu#rTBZ~>WF{M0+_Kb3RGKD5MZ^3CAlXU z%L}*~1=I`9x?BP==`4af3Z@@Z=L9!}&Ui0MYn)%Kj_4YQ+8PBm6yep1;u-}v5b+h$ z4BU>`5U`zPMTLa5N4y?}qZu9Q)g3Jwn%O9iuy-V&jpiQloRhbP;f+Kds%2m$v#N9@ zQ1|AXIACGcfQXf>Ey$Xij$XE5iSbO>ygHsijJ3Frv9?F5Lpb)dB*t3T_cQ|z7LkZt z-6t+Jfn#jJiAb<9WhrocI#Z*4)wVrxW#$@^eB=UUK{Sm9<;RNKSiUZf`HwSujY9lG ztl>7e{^II#q&2~Nv&Wv`J@Lg_6fV)C-~}*|m-Vu~8zcTiYxMgoPdih4tyyIq#}Umo z`~xkeNm;0wZ`~X;-YI1Ahk?*MqCy7n2Z|lFN0Yk0x7^QIRnZy~KWiM)mX}GNgENj$ zA!LccFlT6rALU3SE{xdE%Pzd^vr)3+zOhy{e>f;;#lWz;q>?q;(9v?19jPA<@n$aKV$9a?rCJSWj*!|kXYKK9JIApUIA&eS%Qk4ja7b#R}H{f&@{2X z!?XKD+p=Huhug~0g7V04j>CP?I1aPObXa1nb!F5kg)A{=R_qz8T!kz#*19qd<^!a* z?ZAvmbsGh8$Pxo*H45UeCC0a|i~=37#6;)I?ZS-8pBrtF3R_}G66IE|$Ir_W)U{|26uo)T4QzuCpOp$eau|T7F!*PPFQ*Hpl2+nYbf&vKc0s*D1f_YJ6q{729t{ zT@8XYHuoaz&A0aov2I2M84&^P?y@TLsx-W6^idj8XZvrs+j-{nrO_a@e)V)Xudz9D zT9&L3&&mJ)z_}4n8QDQb^80$zuW)u&!?v+C;$29w|~63 zdi7uY^zZ*Es?9C_ufL!CCwui@kw8vQfB(NmPPCyHa8b@=F~cK$4iS)}z}f9S>|WQb zzi!VmPC+1`BxrK~i7xJb>fc~j{=WJBB--D2cgyGELKy)Z%f5`XG2=@;=4JE8l$J3m zkDr}o`KO3g!xHY1P^Un(FE~&QPr6|mfcUd9bB%~n!}zmEps`VE;Np%({V7UqcuY}h zQ%Z}4(})^^Ep|&7H&L2%(|hJI=oI^y^8%3lnk7=C`xvfvVALWL zxWl4NENNA^RP{TW@syWa7ZHZnOY0tkmD%fI5mVJ?PH3JNk|t}WyqTYkIBiqH-W9ja z0A4_$zp51%s1yKXEhT1KQpl2)(HOM5L?5f{C-p}nA)?4u7F%e~BF%KXec;|jh&L_C zEVZpY|MiL1e-m@tHIIR2W2GcneB6NMPD_7QjZZxLnyh72sQJrrKBZEUr|>44l&pST zl8jMRlBZ9|hKs6Ob1S8gSsFl&6vCC8)mwH3p< z#-oaeEpzvUq}fM4I4sH!0{UAe7uzzfRfl=DS(@+vnjd%C2lmd}Yb$8l2iDyKdtQo> zRaS+pf<*}$Yk{Y&hHO0wP|;fdXd5Ddc{_Mg0B7)s9<9MeS8ob?A~8+>T$RO`9l;lx$Z1L?$%1UEtmJv-b|<)q_ZFDplD>C@dZu@Zon^44YfgEX_@1~q zFi}FE0f?YyIW4H${zPszRGp#|hR|ey3z^<%nlBbKt9QYReK{04OD3oY`c9i&bm4I# z?*+DMV`Rnem}p@oi7*-cfhM<1q~u#=Hz(Z+x-pG!^FrQnjhrlP!5)Z|Mbk9aWwxc$ zQLH+2rz$m~VWfU|i2f)v{TKW$VKK6(hI!~B?t-nGc9vm})4LU>fUoiUVr9@wla;VO zW;>I#5XBU_fAogN_UiOL5G1FAe70D`b+S1w7HN})y{D;E?H%C-RHHv`_gXV zyW6U_v%x6)NH=-L)cp2e!PaK()OoQ;&)>hgc=`6#e3Ks1WO8H^t>bV9yGC>8Ft5{F zv6U6Mr&wpvbiTXc`E&A0s0v!_(6aTJK{*XbNn9U1^2%&(Sfcy{-S7@=En6(YdN!I+ z*YcjElH5bc!JXFKz^kvUHN`v3Wg){`oybjI$PHJPt(bOh2lwFQ8@d{Sj_|lMI>6n; zpG>8)I1Bfv+xE+~yq}eFRw*{)!PvK`1>dqQ1Oi4c*}Txu{YnW&Uie)L8}I+{@>?}z zH}b^lb5crIndyn~K@B`O)joJfMd3^$7p7Lmnbz+V3cs z`hzz$A`k1aZSkQ@dqqojPvd#EdERaxtd6E>!IZKT)L=<)i5FLw?)MfnUiW-(U|4l#h1vhYg$R$6a0`s7d88hS@0`f^%Bj~AJ0iOI9e z*%Ske+|h>=bG*QGTqFZgOehXcKrfcXwD94o&JRGbDr}j3BUjuu0@LV&8%uex9_=It zeerUTQrRm(7g^ZT+C*sg3VEH;6=Co{&Qq@Rf64Xr8}Ou5!sr6QT>#&Lp}#Jp6p6y& zcNGcUsK(#kJA{kWHU#zR$C)PvgVm`^zrkRm1tT|Wp4?t}&J$Va--P#5v=L{-dr^31 zmUT78aNxaTIhGa7uFWpN->6bD;e(BU0T;l^RkN`a*x2E#17RnAVJJs+aAAA;<_Q`m z9Uj-^!tTw%i$S~^%J%PY6}|HkM7&dQFI>~b89O;~PHfw@ZQHhO+qP|+|Jb%|<0P4S zpKogBW>@$Ab#?W{s&%>UuZwW9;vP^obeNsj)vk4vmxHz)|A;4nmN#L5E4kzEY?*)o zcKf-*c78?1xfyHzfuR_6Pt(!@xPHvGr3JK|GSxLHpk}sCgTMd0e+zFb=*7wQAl#Cl zx|q9*s`g!upEha0W2LO7O_!hQ4XGiT$rM)ifz=Z1s~J{@Q>dY-=1h{PjzZ+|e7vmb z&rL|{2?P_~a#^g=!D=xeiZM<>yDS@X2}lCySSl$b5Sa;?Gdb1nS-lFg{OR z-@p67={tsRYqL;(8TBMz??e%fO))bUaWg9bxvPk&DZdKcYed6VN6*!{%@tBAMKI?o zujdky0e%YRkq1~>j4_QL@5%AKzq*99}ZS2JusUjl$A(>$0!Y zmQ#6aMr(ab z&zliB8{JYhsWto)l+q@B-=jiTCW&3u`2hAM1+tI-7Q85My+X+pzlk%f8H?M0oz!!X z|92nDEt~=henUOJTSLfFty)+FLA$-O&*CdqH>X=aSCUNscjLY&A@t$&eF|zQWE`cI zya`0LcxZ%Nr+Kjesz9F-8ZE@ibK%lkU!z zcFA?le_eLP(F_5bC}z5{C9D(k`C||GEN&`o%HJgNLGC1L%@0F{(OM7qH@589!yVp# zQ{9e56@ejmi7fdaYLl^s9Zu8yPtA$g+1H#$Q?CyvpBd0z^R{@W2kMl|5AR9_$ZGLay6vA9dfNlYUEI9Y_01r~ZTGVo@%1ps@$Hs>))lO(kY_zs z7&Qa?K-$>in#jn*ymxFDH-@m72rw1qz4(FfzJ?Iy3lHH0(|+0Wyfb>^v1EXqn{_OJ z2dzb({n;_3U|+w-QUD+o8cdq^ z4;K`$dgyk8(@Ygz+ef!Q@!aWL-gLf2#$D}8w_oTsZ#KU#mv}X|?l?Ew9;q{7>0KTz zgxfi(MzadB%xWq*h+uWz46k-VAC;`3T&XKAPm~?WbZ7g~nVC(~K}KmJ#??=8h)vz#ozZAb22M z)JHvFEU6^Ptk3kNt_X$h@=o-$a|Z>eva4bf7#@u$;}~wRgetB5U!{V80XE$~oy0Yk z#HQ{vOC0=rqOyFmQwJ4nc>C{r!PDH}g{i%~9Td%jMpnT&Z7>E?BK7Um73G?IJo>f@y zgks;?jS<3d7$7%!=B$n*sC+W45G67O8}U6>&HELF)onxbEsgaf{7Y%FFr4tU&kBQ` zKTglTj=s9(Ki96WzmC4VPfg*M!{wT7zZ>{Y7BiT*eX>aC8oWE^iAG3e0s}i7{$^;u zoSx*;%9A7iLa7=2!$g40S%7{#o=&~w`t+F#e?yUr;h(C8C+6VZfi$f3nLIHmK}_Ap zUZPuC7w9-Nl^bCos}+>SX_l%4Qp;OvV&hB))v47&;9!UAG6RV>6=wRth(!KGUZq~M zKNIh%U=vl%1fh8yMA=u>lmtykRf{1w%z@C2$JZp7?Xksd4t?hVB}tWyI9eJ74b)7C z60v_< zW>PM8CGyQ+x3Al>QWT)VR^vie1FNcGEi=?2S$&$ndn6@+>9mz=h!?ud5Qf~X1y1{z zlXd=!F63l%;*8M5Q0+v3RAs9lW|dGQ0RX|u>sZ4WIZL!7Oyni?IG_wLaLg{}^n`(F zIN+wGm@Bju`^h1A^AaDQVZ^4s9r=Rwk*hia>Mn!q8@Jx6rk?vzzzlfc{0!Iw3&0*P zN2mCEVQSN&RBA71xtHjV?TAFsDXCBU@`CWKs3?HPeXFFGA+GFo`~F?1CTq9#;_df> z`mAaFvSxd0RR0om`;w@2LuSX4{2RW0aW<=sPhdsuw0E3#;wc7W-ZzQJs*F1PXs-veI)uCBjdWkeE(@xq@^wgwsCEp%}(>JhyUOF1cIRtpnL5lsNj;@CT|881AwA)%Uefl51kh-s4KA-FpD!@mD0JslvKv^_gX& zzBL3>*Lrl;p+AY(@j$rRNfo7SJ&BTGoMBf1Tf3>E`zWS1Df}gMt$Vi;i#C~&PD@c8 zoVA=ak3@CXib@ljVZWePolU|-yMQy_J!bnhf@oFR^M+__#BHyk)EQ4Q(;JEL&Oc^S z&A5#{jKAvWIuN>BS#^QBl_C{*3YkefjJ5bpImz6%V%Zg4b)b=gW%+6ik+UU_|NGfY7c}D#v^Xs@-XKxpIoR{^JpmNo%N(Wx`i^HRxCHP4CG5?bK z=R2yj>2I>_#FDwWiQ<7d|1?tYDQoQe+;7(1tt{g6FP}ASAT<&7y-F5=q{VGjrQ3$k zzFq^omO(mS$ZlcZ%CKk^jF>73dQZ6c_bteuxP0B79&@%e_@5iH=C|^xfP?A@`X083 zmkaugzAbQJjF-G-D00->#idz*!yoS768^o>oHuDnn@W95b<%T#cAG875MKQ~@Id_S zdAIbl94k+^fXx2-iKG>Tign~D#Xh#Ja%4(sCX2lI>!#=ltY%=1iF+4kwR35}A}8ah z1CtsP_o z_MpsUbr#Jn{YG{1wH!*t6*J-J&E@hvHg=y75-yT`(Q_!>YMffiXQsY3h{kIA{XY2S zq20+?#+v(pQTL*Q@(fm=qJDH?X?wzYlqDu$nkay_# z^&>n=pDk6M|KjX&r5c4%&V%6LVU?S1n9io*^6&O$`kiV1K-s!#zsl59Dmg1_ro*WA zB!VSN{h8EvW4Oa;-nhnaJs)%ya-QW&lM&bqwkG-ajM1)BGB*(|mFo1gtH3;70zSMz zUxTEY%t941Vt^-ni|kToR+CYikzw6ws`x--3J%ns-{;Nb|G9#*6n~?2`VxaZcS!ed zXe1#xahp(=XHNhAEBSX1XOjPCt3UDRjV6)k08=_% z{?~e!VPI2!1M>5Y^T(Mkq5jPJ_D&JsnGcOZ%8F*HswBGy+GPdo49BfD_;VL_vY{wN zP2S);M@t_=cH#qF&F2OIUtuE4R>vu6I)&nl)z}=ZU_W7@F)s+$G0tsUhv!H);U3#C+H=fwsa)?T$5F`6!8)67lFi4AdC-OgW$SqdtS&Ip`Yd3-P5L*@JQ!ZXK8UIj~q2O@v z050MWwVoE4YJ`+^W{x4~8vt(9a2zKkwB7<-rP>A^j$vpS471yH} zm}T*{=d{a}bugW;97GVytzbrN;Qql^Krfl%p6tS9Nw8 zQcVZd8(XZp0zZ!ZQ3^c}+#*wifm=)>OC2H1FmXKdZ z`eDrQKiLdOZAX1Fn)TD2G_eTBX-bMiY}aYmvQZlB>3~GO-rj)^oq!;4guWSn^NnkM zLC4cc-Z&rmvd9`p9^53gzg!s-(hAbzZt8~q~mAl0^5e}dc&rP zK330jqYO8}UD2Bb|18Sukf*UmnYteAj5Oc7kma#H)Ic6-j#7}R*66U4ZLQ88v9@Ta zX7n^)+l*DdJ#>`S1JX<_(Ow`8D_}54nNVfXH+)#b9Y(mJSnmQlPc0|GG|gN}Ww;%; z;4I7IWhV*Da!X|=H=}8yY&4;#$F;M<@{~)V@l}~`;iUbBVs+IZO>&CJ0MNCl#5$Ex zltTwZcEDU15>WFg9nCQa1=M~3k>WitJ*;gtL1bC?1qU+>C3haah3sunt=@3me7CH^ zQY@h@NA+4}A$_Sk;+h ztIxQFUWH#GiH=a271-53|P)76Q}ND%qywYD2dSlK#BVlkV`degc-LNh+nt%?|mxhT8XuWtb+Fy^e-L8_Z~PzZ<%{8dAnGz z(B9K0e0B_1g|G`_5u`Gu!%-~Vf@?axR`}BLNJ5+&B^IKY-&QLwL4@;uzd*{_ug2!c zt(`050+Q889T-<7QZm*G9Eb!UB@hKV(_#%g0bz2gJOpEyq@oKI=EhmACUoU4iTVDBpk5k)72&BeUkUtki@a z2==4hbwH{^Op4`97ZEDTO`#IlNmaJ20(?X|VNt^C_-_^$)#vxsEM0Ai`Q7K^h}8@x8}VAX&f>0JF67LO6NkNQ)L4tP5itd zY>}#y3@6jjhdeB?2hy&D)Tp;H)a!-AKKWN5<7p9<6+q%0P~vqAx2%j&4Yz5>BVp6n zgcO?u2+2pE0Zf|xRqCJqI2|aimvPKMPBZG(-m@(Ru3|yMIX^HY!KwXOFi8wl3cB^9 z!y!p%*%7z|5fWsrD60V3VBO&2V`xrbl9Q9hAZfN_Px0Mie!7SXoE<0`8ELK8t@}g* zl?HqYOxE!A$voGI%tEV}nO<=?XUhyPMRQ$TcgTPw^v{E|KPxXW^x9Hxh+glpxus@h z`yS2{e3g*_&`}v-KpPLRY4EQLM2zf3?u)(2;4D!=5|+kOg?TXtdZ4L{ zR|nI&KureYWm3dl)(T`oG%&k=Wd@K`$5-QaNc8T@1#YabOgASU{^HH2A}`l-_9fPS zvRlta>qON`Wx~F|8X8Sx#^Io?4uPgL8%S=P%zSG_BIsl?uR1%;ui_syFrBM2EMMp4 z{a)ZtGnDM)-F1_`Jd|SJV$>Px^mAH< zkEr{1&5zeEg&3k2EPCdg5Ex@?nG{HdY4N|O!dZrCo&;2u2{F^8-~eE(#tCxYH{%o0 zXb{U(<{R%e*7gDi#+I$Q%8(3y8-?y;BN~OxFoU_~BRCqBSl(PNp9g1XEz?TC9TrV? z5HZ-0X`T$n7sid-WCzH3WB7dbgg|&?!rZ*dfrSTz-XXxGcN48r8yLN^)sVE84UwgcvRw}wyUV1Y%{0*(y7N%(@Cmt*(J#Dt zS~SPM+T!lR(E_6-I`38&r(zw>o31(W@<3%g&wM!qPf0+mu8vjFEOp1rW$#Fed0InV zr@Lkqq&EeIo}-BDEsb`kdzp7BGizZJIA>0*tnxz^!0#h4HBc(942|D2_Qkr_2Iv06 za+Bfm^^oE6HIU)@E?H_n<-K@LmgB#8#y!H*oncZS z36W(|FrS_GW;k z4j9uuwg|hWCqEIt)WAGJ8;CO9X;`L8jcy6WbrS`a?_}QP?jc2(`DA|NWWI)bD|{XJ zuZ^^lff&F}Dpp*PW$=q1v@Y1KZialiV&U8>0j6os`gvfbbWC+-vHCPCC@gOt-RVr~D$LgYvu~(l_<-uGD8LX-LZxWBk$G{+1d@2&pbr14A|VkKo2UrD zLVa~?Oni-N3pBtqv~~;Hi9`3{zEQ^In`iuF`;;KY$EVzLFH&108naAbnRR0oTLPzE zYxuST8)rI{D%de#3EzrjX5yhSGCU<#;~{x;eI_Dr)qO7 z_RiMgH?=NN0METy=W0Wuce%h&?i(eHWlcbQL(Js%*Z0GH+LRbWAA2o`teLlNpsumP zkPREU(b$i-tT{ZZiZOpD1a!7btpBF6jUpDTGyOaA+Blf#>*&Qp8x_+wo~YPH$)ra& zDyCgFkyi~NXdXKPF-o^%@5_wGefvnpos>O?K?)Z;%ev}mPpcyU_L=`-JNHq@k{0A< zj1SKG`hgDvUTrRdy6E8|&TO6>(Fi|Qtj2sRjsK`}^jnK6#Ye&VGzw>@>1IJQn@&`i zrwdj5Txq*dP%bQ)Xe){MUE7|pZ)brNBp$`~t!Y+_zpsBMm8EVrpBp>&HV%TQ-?qn& z^nB5U60b0L`*Q8dOUvS206W6bJmonJdPtP=C6X$Ssb&)09i}V*BQNjfI#<$D`vhHE zM;nzw0o}WKhbdJ|FL;&N=FrG1`$`^UE)u0o;O*c2n0gyOV%0cgLM1pS@P0Y|nt(;d>Froh36~o;g3(;$dXtPDG3&WRpt}7TG#V+tD|#X)@HWUOW$Iza$Oa1D z*}RqfNM4k4av>|0>rM)-W$LIcMP6fBqFM)|nQQSEjcMZma!2V1B>;;Zy-BCNK8?&4 zJ|`nNT4sVHYbz+V1A_0fZnr+uA}B7qd{Mlx0gm1s#3il1(j1mGh57bAvjCRilI_f! zl1DkN+NGoDL_iUndCsHImP8w$yzH6PERJ+p2Un_k3z5v}`}|mIyh*aLSRhL`wtQVt z0W_RR@fxylog3qtp{N(su4rrL$hmB5=vCiW0syTg%Gk9$6wghC;SXi0m?yY$VhJoX zaOe0u=fO^W@G@LYZfc>a(r3tP#lwmV5LcaEfuc(Vf7bs~P)SG7E z3L;U`g5OM3GJ7KAin3#i6V^AF7hVl+3f~vcQTsqrkomHO6pW;ywiZ-9b&4LRMl)xF zcC%9FM;NR#<@=*xk{zR^mPP_QKX;uvehGbt-Dlyp>7~j$SxgFzI9fhWFD8 z9Hp1s_my$hS6YpNz@Y3d_6!q^E=GVwEKG*T`Udt)oFk@k8+-O$q5821^QMip&sy3i zxa1t&o7mG1ZgUa^8>Jh~Kat$9Py}LRKqP&4ktCv9`nnT3B^?+rw<3{{%zG{1$qyq0 zi;!S|j1#F4t{nLm3L?&d{L1mM^>t$9v&0k;f0C4K2p8QRcD-Ljq2JrO z3aO=hu3=3OO%=8U50bN-&$qH5K=_im4ihY+CExl&83V|Rz%9SGc}eOs+n5boC3uh@tm#}x2!Z+ zfZNhGA`L<4ALjh#!UjitawIbqEWpDOB5=%Xf4PUtD9>?&eP%j~Vi_I+<3Lk9xX*a;IZe!`TY6OJr>@R;fRZ}LtoyyV$ASh^TMU z?D(9*E(C<8SO%W4GuuCbsRP-+lWP{?T4Fkx5jwHCJTZa|;ir}&Mh-ey0y=hM!6R?t zEbU07d;Np4@ejPZc3!7)cO>$XdZ}f;fCxEuUPS+~Z+k4j{00~~P>Z1<&Jq|okn&S* zCG@NSN=~10HrK&({_--k`DI8mKxtwbnw+KV&=}X4ijNHqKRYU3b`)?prs7j$E;9-! zJ2Y;*lNkjN@iU^~WB*ltU}XMjjEEg;6HY$(Pj+twfn!RxHF*CeNWy)RE71 z@RLLJz6@brxOvUsI*`eAD3jwzX0~z?JIKgUOlGF)8T!v;$jq1b|NEHC-v7^j7>}IY z#AN>)S0>N--_rk`$m}`<&GW>39kMk3a$y^eqE!N*5N{cV7HH|qD0JiKRsJ)ud-1?r zL@ozpQ>;B(k;egC;M$&}$m2)Z>>sXe@0cw01 z$&A)qth$^v+ldU$xHI33n(UM;HA63kK?c;<2<9BV-VpzUzLP<*2Y_fw_p+mW8rc%;@Y~+ zx@W z!P|@DlFU9L$(y&H{srAdCI6JNlamg$7ky|y*tUfzCYt3Rhd*n%t5v(Ju+WM!>er5a6BGwVd1pVjD@^FybWwzz0&>j_`ls5rB}lNud?ANaciG$`VN*5z~X6`xVHblT{X2 z7ka?HpO2+y5d6H9B)cTiL

R+)XU26|Zdelf7{98&W0o*ki3<7X7bl^a_;{Ju)P zMU#KQb<^~EMEiLzlB2c1pB=nhTHaZ?^+2t z^|9Ir#iEsy0n{Q1F$R)l7G_*Y=}IV1wUsY0M~d#SAl?&eQfPC<@u%MIu^qm+zP`S^ z-gs?VVRnKM6>Me)ON+SCPi;uIgXk+Tiv=HBN+!x-$IbY9Y~hHakRP#on%*7~wW8#2z-9SW`1^(+76Nr~LS)4%09 zAv{ZEWIHl4DtiQ^p4VkfnutK@M$mZb>`f^nC!jSHQ07c~;Q#P45ja19{gdLi>q*;x zcVI&xTDW*qheXz0ARQJIddEd_IgpXdSFiZG>0T3XpjXK)MZAdoIQuPldAd0B&*sZT zI79e0OSYwkZ3qWu;Z|E8lN2h!!BTX%+;vLqE#K1zCbEXc)TJ4CSNpgfPRfH2T+f1; zCE&4&1qb)L|9SqZ)|Z=`ovp9)`#Aj%5FN2P@uR2MeC}<-Ug_!dp~k7 zvjei#0CRzAp?(1fZ}Aj#-Ok2MdX0mb{w1@8@3_kpns00Pc8 z&)69WP04P*9MMQ;y5P=FMk}L;Xa=`?ws5%UTQzlJ{^QopUT(X;L2a3j!F znymr5`MCPM-u~M>n*4S|ALqChOS_3pG>xa>W)Vx{eMZrmIy9|kERaN)ej;-(AuuXx z&_G^MnWLTGbq$QtNpV6eC&tjABwB}kt{*PPNBHCK!}aksmyc%~*H_`!_u?nlo3Fk* z*ZJ1`>`x!x$MJbwNYyoYj9Ibq(EafVlc}C_Vi4iU$~BGf$0*Jmi;UuXWR$5Y_TbUM z!kXyZ2t^o)1j(96K?!u*TBxnjvS)DAAjC4x~r7z_e$CTGES)*32Vj4`IGPYTq_O*7B$ zk6$drqYVqg3k%!Bv$}1O3E0VjQ)*&im!yu%>yx<5A=h(~h061D`qvei1xYUP#Wvos zOU`?0oC|ZDotQt>VBtPgi}Cu1)rnHzoSfj0tDQZ4^0d;f-fu;&bubxVamxaA^$EKS z)Fg%BMEce%s*NIA5mX7!*%29t#j}SbOA-2htO6`1hsG#IO5pb$Ds!U(Y@Lw`nE2uv zt6zMPlL~n7;fexSqyn-MSdx~B80ER4_Rq+8lbF5-i)a9oWRg-sHMW+3%^pMlhcwk- zxos|ApBE4+vs70+U;w*m1CW?8sgHjVxt%bhifaDH;ic_mjDi7ll_dRAL>Xs>`240vdU%RTX{QT2g>#b?AGTF|0`-vhTyyCD1Pwb zMfLGA@4ZBm%=+xEm$Z+`@5@bABrH0p=_DOf-cE~SK#jF{wcTbiY5A{|@t6P+GNGZ*e`au~dGRe|2xeTpXx=D$O zNROaO#!64hRLNDVqM!UF$20C=znSf`Y)w_lXoNEl4@bg0te!}H-R-a_Euuk)+?qyC zWI`@RhM=9JCJ$H^N)B0%$#pTQ5SKz6qwWVy`>nc@Qm;z}@}@(6rv43(K-!$kwUkJX zk_J>P<+anCfq{B;NqFN;d4xuO3^BSTj%;&BxG45G+VJ;_5w<~CbPcMJ#cA(QWZ^)F z0xHAMCIX572yCule1qObEHd*AiJ>=p+aBJ+#)&u>w|lF|ev@%hTGeVSXgt?U(q+dS zDBP*_u2-_>uFoEcV@V(Ey6>d_QyT$}?w={F_e%hd)+2QXGw?Gs>U$LZO{;uamui(k zdZXLGHhwq!+GCeI8vR_|a=GMAx8K^had8^UqXxGa-q!mh@%_3(Z_Ss=WK=ovTXiDq zVqyhptjkDTVjN3flRFo3(t%YaxmeWK)0+l^az&|~jb+qRGiXDvu-sw}$$C~$mxx`% z?_W+#4DdlAqDh4$BpR})?Mi{%a(?##Tn)#nDykK_Yj6jSs+6?^BE7M5$TlH!pKBty zc(qaMsppnB zvM9(bdj!)|RuH#fBa=S6u0esM;E(9UOz74g^L%jH<&CuC#LSc!$> z!NCc{M9)P&CJ#Sf6nEdk!Lg7YK1V*+_|5adcz}MqGhO_KrH8y1LBYYqLEATI>uC3C z0O!gu-HS!|r|-@MV`1NLun#{TQDZ((zIWJLjLBJl_VqS-&eF0nu+mTrC=Q+)7A#R44g$;9gzkQ0Esvk};&!Pv^tvDOZ8 zIXTEfn2FjAb2$}+bC9P>e3IY`#}`f_y=%WrWq<-s&ewTNoYtcF*qoTTivwxDvbl?n z+~QAjA|Cq1diqfDS<};1-!v{`xs*Ypr>kj}yiPqBeR_2@d*O5LoqC9%cZ#u#=_BE; zgBThHz+z1Ki@8K&8Z1Bgc6b=X%}#DfMfmu3jJzZdSHU2%!;n@W?nI0#iMY=KxDua> zf(pV#;VH8+lYKYqDNtm@?YA;Gb@Oy{QE&td9w+I-DQbSOVHwGx6(6THvj6W1M=|tC zY0W#InZ)nf$>F`Nab{>7&3o{xP9FA;jtNeu!T^yUAuOg{|o% znm8mDJHk9IeKRUrPz@_{D1~XMgA?-#yQBnmO`#~UXoFuuz2)YZNjnN#Ouydms?0n$ zhVb1kHcYQK>{q9Qlm_J24Y@0Cx=kEu=lDZY=orgK9{y@U>Ul7z2G+_Q9!`g>RixV$ zA%%0EUER6=xOLVNdSC@(ApN&F?MJ%8&;}8txdfu70~s`sSkg~9E5twADK@n<3z9?2 zu1d{Pk-75xU(-)iC@|u7TO?pr-m_tU-fx3RhTjjs#hkc)M$~k8_@0l)(?Rv0HT+rr z>9}(Yaebc+rTk8FbE+lFh-FnV(&YItCaP3)`G0%OT$`ozohVZ`+|~@I>urrm)HWcR zPQHPJ38rUR?E$%tw2?VSCMTK$&pJ!X4b+IBsA^hHokXuTl@paIQ0)OaPc_rJ^x`p!v2<7KSWw%NA)G9rSS>*rs@5>HfaB=E!K*IDqhWjRPKWq%E0Q8W@sWo?)o z167-X#+$9vZP)wU?kHr5SVfF3K2(+fK|KQgVt3X)e)YeW^L|7UV~$cH*y`WRUiE4j zj+}`sq|oShFEP7yk%*`WOJ7_o$gxF1CfFm=q}`cSG9JL2t2dKl*C!cj4g?LeWZxMf z8?mnDlDlMWHXhKAl7$3lOCo2QK9t~-4-hlpgX~b&4%jGI6|Bs9R3jZllG^viXJ~e9 z^1@nV;{-v>&gVKuP}Aabcj(Nf&GFipy?F>=i0D17?NSo40Hck{eS1>NXwyuTGAhG3 z22wVHaIB_F!ww{nuGE!gc6wdmzs|T7E|%JARZ`{W+-pu{$W5f5$M@Y!?)^G>eVqK4 zV0Cwpozx6V|4Pt%t1fT@z;s*N0&N_0SUWDjvQ-F+gCz25b<=aSz#EqHn~GG@R6yn2 zf|g6-F||_V7NqqqSNtBKYXlNnWN0*9Ivyj9d&7j>hgK$W6QSw0H+yY`orY2SyaUfu zWEPU;d-!=T@@)w_@{&3u%v>>5MfTx%9|~m_I)B(CdxNjQVBFIZS&_Whw;OSZ1=BQ2 zGFNf;yVlTy_5~BUsn8ri##WYLUudIMk`FbmVul{Y^8V6sNHz|k+|}j%jdV`B=K;{< z8hCS)p=IU%lT3+GY|Gidq?{!4AW|op>awBBT^^2EkC~2(!h#Y3$&lVI<*&c(2;>k= z*CBKgPf}uTjrttO>WYQBIR?(v*X#c>e0@E9ESp z+KviI6u!%ROOW)iQjq*ymx?h4%f&uSoD@PaST4e!he~jrD@v%R?bB4dp2Je+OSM8J ze)nkX_F>x|63dy#{OYbjAZT5Hyi)Xe_0;V(5RaNkp}YBB=y01HG#9?FzR>nZS^bgK zfF5vuogSTjmcE8jPdB@FGinQxHcIH1Oz6Vhkdsw~5oEEUYleM>crGA=Ri&{ypsws5 zVM|qPN3PP+E0)DQ&AOD zu$M5yy|^{sZ@bVPnn@*Wz(HlD0eycCvj27Nz|e5+>=St_PC5UbopR=vqEzrODgc?k z(9pCUzc9B?xC0lqavWDbj;kK;t-yL4zI+IAjVTw7U?pVYG;otkUFCu`X!Knpe(?$rC~9#??ggL=Am-`z4}Nl8 zQcKOc7F#JIKyR)Je?d**3ZqkN_MFo8IXj zzb1DFNh|o?7yr4%&&R{X#rt`m$*!+6 zKjU92{=?tI)X(_)>F`d>`OAqszOaDdD?4?yP!N++35|i${6bGKO@G;y4c|5A>v`rO zEQ>>J_X<_P7lPIa?aPxhK7xKLZJZvfGzl*6-NXI@mD*4bsINS|tEL|Z7!awg*Zp81k5hffDdL6+k%q7sQry>cp$pQ~8%EwX> z{HuWkEhiMlE*nIvU8(D*=e^x4F8r6$hr_YMY6)CyazB>;uW19+gU1Vo-&YqjA!SON zd2!}~>3B#w&`r46x%a~ZXzqf*rl=K*vhP=&+hRWa_uir1l>Gr zZJp9sJdmWa=+m7yzMvQe$p*=eDPvX+D`Tff z)oih&$3;`BZJTwyo?tJI^e8WCt(mb41*ZUO#DN8P`lS@LmM=+Lq%HFsOi)iV4KZbr zr<_u)hADOkMif*?anyStF&olI!9eIiJoWTq3{=YiOY^0+Fr$jdamvA~%&^zreiVyD ztv|BJbOicRju8_L{#EvH4N*{4K`}7PIcYZ_noOe1+V}x`0w_~(_Zait{=~PzpDXdn zF!E%dQ;)~;FZufzzCN64&xGzI(2a*>L!RQu^?#s9g0k7dqK19f&CM?axWvv#=BB*^ zG#$!L^D#{w&}^f`QdVZ%T5Sr#1P!Pg3sYc{DCpP}c{>!*_^nS%&L}w4&+qq-_bmVe zD5gW=Wi=oPWa)NlwgDxcz&CReP6>F#Ri9tsE5gbo_4Va5#++hF#H0?X&oKR=DQ=2b zLzv*-_xr`*4pUpw@%`G0tP* zA=9_##~_BXwtiU|gn!l7*PH7jwTh~-kYJbsS;UNiIvV&USQo3&J?F(r)wWSK;cgR@Xc2mZru03l-%GLai4b z&eTj03zXQ~Vz8&dI?fldeZ^>It6svqdDv{Vi424JSqEdDwFqMV#7`<}+Mf<{1#%8V zNZW^jmayrrPS#5c7Xushyp(bkzy)p&jC5kKF`Io2VWa?ZN`iJ6=AFMpZLFLN_ZCrJ|&F?w0ri}e>raC~(WiZ8B>g3yL&>ag_Q zf^%L;W=nTCjY$m+7jex+v#QqGcLL)cY$&uPc?C3Qg~r59Le%<);Ob)+#+#zc(;0%GccSvGb)CO1juG|}$4h7b+4@?y z)J8f3YE=bKt}#+`>tza73(TmzMilfih-?W;!9WJPF^_VwA?!7M|F~3)TtYN0egoyR zWu_UJdkaR&W_ zzZ>xR_?Id{o;0ro<}1zz;{x}CfpWs}-M89}g4CqjiaCT7DN8cvono;V22CWn?2MiF z-|TmYSNOS=`_0yYo)mz6S0SXjC}@P)J|a=3W#R%OoaMN1!>a0{U5^xhJhA19U?oRl z7xhGt7YuDExYtKB#MSUgK@~xi=VN_n7EH-aVP#!Lx*libNuhMR=!p zP(VX#az2kp+drp$#tqDc?X$!=#@R8U#YKw7Ey(5RcR_j_Ijk5(Ij)FUkrl-NiD2TG z@of&uNbwKN()o%+4q+%e_#JjnLyvAg`Z}N-bmNaIlyPld>aj;v+KBe1sDt_(bbG7Y zZ*afgUwAzvcE!<0#gFlO)lx=3wQu&olWL|XfJ_;(uzsg@y0}y3p$k<9qYCCWC~)`L zr3vv3{5k+W>)g&A%!O85mXn=~^#0DaoY9UP{BRrpOS_A2jVAw3YtbG1zTrAF0rn6I zA9|D5pWF`{9z%Vhk7eQ6$;=h|iaT$Q92zuUO-Ly`hYzCGOD`>Z?T8 zRsS6pawHHVK1o>Ym|6%83DnPoG$W~vEL`FB(?V<0gixr@<+S%u(D&xm4C|V%kNOVE zle2&Iytpf&@a3rr_!@cDyV;pX!!dHC6f(6>V2ChzNHb%75L6hk&uN!ZgVa+(vjy3k z8pzJL3Zi(MaAXWd0s%3<0>jjSy9k0Cam6+_t`wp7T6B$e;uIlb!?h1}*b#5SD4eZ` zvNI0)FV3YQ6$t-<11p_QlOOKsB{F_C&Uk$!#!`_WC5Ov1Cy9o`*>W2u#^DEXle1Yt zbt5Y;aGmO!w?RIpyXEuHM#Lt)JeSsl%#g_kV`A5#?Ha@&BYKnff$tf6mt*tM!9T_* zIVRow=llC&MJ|7DF`J){uDOqPw!(P`WeEFu8>1u%NANM;p1-)wHu#TQo%(R8zXm?= zZ$HI?<`9BDMSOS>Jb^*uP8&y)}zke1ZqzAS5WL*C&e82fE;!9-+ zN|d~v*O3Ptr$$#EboD5`?;OVTaoP>-Rlp#D0v~2LTbu!AXloP!c3smC|Bd!=jKv~5 z{by74o2LH-3fxrb{vTT4>crO%V5x=zCL5{@*3y+~p z3f!k0B^F&%OT&9n&Z_qu)rd`ML6t)$RH2z3+czBcSfON#tUH+5~~*nVn>7S*_oL^WyqX>Ae?Bv1tsbO6ygl346EquqhYL8 zVlu#S-upn+0W~6FoY~caD*?E1|FrN-|DPV8()Tn3VJg7}#&V=9oyBRQXUrxZjAlO( zK%BrLLGx@w78r}lh66P7k;je4hsa6@BVIJ@E8|Sb%7x}b#Et-IMtel$Qj+M7Gt;xp z4rN=1bTV+3-9iZ#{J54yH{nX7L6nU3jZ2@o-|ha5g>@WN9?jhXkToq$wUs@=V>0ad z@pRV9X`t9z@R8)eDL4XY5R!a@n1dL~wAVdS5YOrOy=w{@>ap`x2vmp|!hcp=B$O;d z#?yG}l*xGQ`Z1QDsgT8TVlbDdLrDxoL?L(^uJo$XTCEgRUlo+G$N{k!<~d`f~c3c)yz=4F9B zj-J3ko{pe49bMF%@(p4A2e)5}-n=qy;iN!HE{(>^X!q{IV;s z6uhEBut2Q=g{K0*nsFY<8XAa)JEqR~064FRnFiT@#qZ8e**uJasuiFtyoWtjclDc!V|ThRv_8{(%Z z@28KvK8fw$n+4Z-P{7z+4pfzX&;Nevy06}2w-V6^kb*D+@a9R8VywJZopo!FUzyXN z%J4!FCS=!AC-&x%VO!IwoGS{S|op;hQuTD+NxFh-9B;gkHW{)4~L@D2u8 znBY_ck~mvznI)_OMA)Z9%OXPZp42Eq5{dl+aA>7zO@$X)CW4M*r?OTh_s!hQkDgr- zBi6`(bGx}lz>mUvtS^SQUTNmoIgMvg!QaU5yddhqMM{<&FEXlW1{Eh^o*#+s z5aP35nh{=?^Y?i*s;97H2NO=A-SISL^dvb{t_G2bvIG>e!g@UkMd(?+K|8I=$a4{e4-B9A4Gm1sP?g)xIRi9oqs{>o+nym#M}6^V7@b$5~L;z zVP$d1g%DT3TfBwFz9E?`;?&NlAg89>p<2qp>B8}hT(&30v)RJcb^M*V@sAl+KmH`N zpq}2SaDVhk1>&?!{MQE5T^+untx*6E4&pXCCh$ZGXb54)qEiNR1-{fOn%~F~@nMqZ zBHj|JB@Fk^DcjB$$n#vuZ9cO6{2rYhZ)qUDG3n;2HAc^`@s$gNbcd>@?{m3{a3)$M zC}9)@qR>Qx!kno|;zSr<7fPr3AkR8&^{Qp6fWvEojX*h(?$u2W&*d#b>tw~0w}^x} zOW(36XTxO?wf8VX{}N6HvhY7ntO@BX3T$yg2Wd8BtE4u;@VNpr)S_1gd8g=rn?t*s zY8fE?79babR%V1J!kXw?!HNNw7Ju1Mc}!hUs2qLQ&gwAOK*ebQiM8L?@oQcrd> z{>eX}8MYsEvS0@U@?P8jQfYlAm{ZCDDlJhfut^0xUluFUqY+rU0V9~yz0$Q+5oFmY zbj3JYawXB#{Dp*-V?s{`QgW-DL%Ndm#-DgJ8nIOKtPYm4C`X-$^laKj&;@lBnX3um z##^s?&nFbkUjPwZa?|*Lwq)O8R@$eXy3m7W%~LH^W~nOKm>=S7^KBcVycz4^enC|$ zNuFqSsMdAO-^QQE#9esWiTjxFvfFi+;6lM#JpykR_O-IQE5&I+7h;oE?HJw&8^Qw9 zgkL5&za&Qn^TCCgpVUzax?ENnX#ZpxX!_Znmyx?}>nPrJk(g_fx>w!6QC-IG$47KS zjKaGJjVSqCR(6hyf7t+hhqR0@Oj$&_<}+vTG(WZc?$QqO0Xq*Nz21sK5b zwCP4_Wt%^3z60CT&(4aL|J!#L#!w0Ty(A(S`;#PGc99DOf@PjJ*eo{okP<Zp3K=2<@`mkm>ahYc}!41yH!<6a02?9Jmk-KvhJ2r7F?^)3wxi zujS7XRnvqZ**TJZmACWl#G*L@!;_Mmi$S@|mT{M;rEyb@E9n?Z?W3cSm}8cK9V(Cc z!8pEuWd(_HKzL6HFpH9*VV#g#-#*U|^9LP~dJ-&Fv3)CY|9yDsWLA4@OH}6?PhKSk zY_U>c?t9X=j`}os^W3Y-CC9_=QH=J5lh;W`ME7Cr@z_$$*mOva?qQOGtf4m%kHIyH zQoa~y$9aEJU!NZC*MKJ*dx-NhII>?FSiBIVpWj^NdS+J-jucg4j_|O|XB}HsXrp0} zjF((`T1i1jy(4w`TnLm;UxIqX+QAdKj~O7@&zL;;2R`e2q3#(2Lz$4w&aOlTe#%R} z>2`Yr9vYWw^O$sQRTU@gC78k#XkvW%A)r?djgldYg?W2y1b;H8kt65T@O+4=Ix}|F z1h1AW$q)N}-GC3RpjuV5pZuZ=a&kvV<(^{metg%KSpAV+zSH+kZ+Q#nE|RuOYiKfz zVB`;`{OI%`b;F{?Beu<}nub0s-egxC${i;xh)EDwPq40&A4Tux&6PCrA6Z?Cnvc7W zwsC>m20BI>zv7eJ^&l(a*;nTl6`ok>9TWnFwDt+d+=RI7@H&l<%#FSQOJ{;^gEMd7 zdfz<4_=vQMdACIYi{Hr~ubE0U3vQ^YuXaK1aS3LFe|``7IFj>6&&f&VC5;) z`(m@vRj=Wia7K4**?Z}9SoamuluF*uJxXw5D}9&0zysoHD)7ESat%iD$z&OwU#bbI z)c&CyQ+>5rFo2oXNJ$Vz1!f9v=z_JN=h(Yw-1Lt9ugvY}3=QHT2I*xnCw8r6bzV!~ zODOoMs(N5q2p?wP3z#iu#X)Sc}} zvriovduZLia}JKSAeVW(A$1(8#w5`klkSUh5Jbh;@6WJ*vUmr9;V0%ogjqJ*DOUTx zXN1(|`lL{T&xnz8E-hh{0o3{oMp>D|Av8f3k&YsBQeYs$DeF#}FT`1ZH`t7of%KFf zl(jU&m6#TsRf9l&zcG+rKz|761>Y2ii7^e3>A<=as?)@~=${fmn8el@Mtd4vP)y)- z2*7lI6u0q2U@p!*wkea3!UmNATzqRv=XSJlqd}!Tc`E%rwV@=~&V>z0ZrxN1P?TqX zeNhGid7?zel~izzVkX-D?bYIOR`NeM_;dV|v#IC$Zt09_V-!6$%J(@`d2r%8b5o}U zX8&y?wO^PoN_5$bRzaP1Y)x`6{-B)6Zb$#(e1FJ>n^iLI(TvgeB2v_m!H59}F8qOj z!Gvgz6BEaCT~gfiLt$t>?*Jo1=cQa5pcTi3Aw8kcULZJ>)U3RNFdB~Jj(Mp zb7a~RAO^d^U|a{yZlhc!NoczvP!69CyD{%*O?<>SxjpH< z*sOiyf~o*~S>kr$^rp$lo6;+JwU@%^q%(v~RFAh9)VsqoV}-WCS%fh^aZ=b~^T3RC z|9v8=V~~GzTcjgEodmXTjOUCG@Gm=pRd6QLp>RHao(k1;qV4(Ub>#^}y|gv=ZOg8X zE_4;KbPZ+qj)0(lU3Z$diHpt(>kf>%@_gWJx7oZsnxt0_aI?!^N3$K3xq;?Ii9K*b zg|1NA%Q=dp%_^`Hhu-gkEYL!Kz(z%7*sV z&g){@xEqB=V{2~hL&GZ$Kgb{G$^lUWio1q=uaByFN?cJq0l5{>l?bl)@@q_JaDN#M4%78lVbL7eQ4ipK&Gxyw_~zt9u`F98ZS0%_B9r&afQ; za474LuQTnkwR`SQ5?U`NGoYNy?_T-v{x_Y+p(UN$I{WhkWHDLl*3pkBl87-82tfp8 z;pdqJS~Sl)SN)ic*Cp9$iVm!9rFUGZKv_P}GfBnAAxxok#?KrBFUth>01NMVTPp8q zu>|~S28G@tecrEQkF6eQOLiQ3GMwx~?{+P~)Oe~%3AbS!;Z-N_3N6_D9%%2b-Pk@$ z&T?`kbTFQ{U6DI=673IuTy7UB{P=!=|E7RQS%bJ$1(eo`l1!WlVo|qZ!I{Ow2JG1Y zRH<)Xd~eEXBCc3CBS{Fqi<_J&im%ZS>20y?f;+UHi!`ZohHIxf* zheN72{3j}IyBV*kY)Hf`I`xphGIZA0eQoNo;EJ!v4~j0yP1zT@>vPO+{V56js!j!)k#BkTyQ%NMOw%~LueCtOOU!xBGbgIGzWWLQIQ83N0I?5CyBWD21D zDfubtHkd#e2|De2XoNSB!S)w7?vzwsacglp+D&&u<`tcSKm^g=v3A^yl#J^3gJ}Ut z91y9>qCoNFZ;Sl&{^(k<`~9vLii%%D=qjDDniAQ@>Of0L@RATx1HLv>^Yry#`n0tJ z$5$*Z61hq-*g<5A4BqheAzhjz^m+^^UjytMaN@#uTQA0b9hbvcG}ugeRo{)*ToDfr z>#v&rDOO`;b zB>mR(EoPR8?KNS6d87?A84?uPj11#}R}YT8YWS_xkS54j?2+qhd1A>laxxDX^Bv~R zgPp-ue?ufb4$UhiZRVNd1Vr&HMD)A&rID=jKZeQ&3EZhyZk;}Yz;92=6J-|F6QSa- z6GxfBVHVVEy*;`1ePMeBvDOZY8f4b*BP>}&j<6&=BA)b*{P~i3+0B7i{PONWb7VJj zr)n}#5OxZ+Ez=EeB(FXmZy%`jhezS)Mmt$UX2mZ}KlT($%IAs}_^C6XwtlDa81<^v zXEe;__C9v5XhFP%eCp`m*Kiravbg=zu+C??**|Bgebcz+bp+ z%W_LXH+;V>jYIn)2|Y{XBZx1y4ZU2QP#Cf2kgTD3X|mh_d~(lvFyr`AAzS-Sios$t#J5?G1b(Ks5*KB%6Yl_l&__6P(u@;bi$Yfk=io2 z-Y7hI?xZnty!K<#2KLZW#Ft{;EOhFlYI~@~gy|Jg=$+gn@A=tJv zL-iK(>!UnPQOjSREM>aO<)8vCZ2(*x8Lfi3T}itCy{t!z+P@860wo9Tc9hY53nAq; z&QVr&&&sR$HMmIgiME3MJh$4;-T+`v2pm8_3DK9uTG_`n)20@QK)+gA;D@eIV=|(^ zCTg@Ubj2K}cqbW%CcbrJIY71gyfd)j)8uSQq@CFP{9o^(!${L#JeBl@afuAQG>etUd)+=@*{kKJaWl^eFJ3-pc`_E$G+uJ$cSPw*Q|R{~Z5EYdyw7U>LnHrsqH5F-f( za>D_f!h$rDf{Zvh!3bS9m1q`g(r%W28mW-t0bRVC1VRIu(!|!jpl)5+Vq#MYB# z4Zid{mS4iit3^~=8KfWcvZ$r+>&0_j35q0=>QZ!H@e%~5hKB7YN3ZPwklN)`U~C>ax$>zY9{C8D;EcG7>I;yzMu5vbDXwhBam>R^#7 zo_of%Y@fAa7X5WFjoIPlo?Iy;5F}~i0t1$F^`5$jrsFgE?f@5M0gG&}E;4li{>tnXn}rFZD4g`sXg=y600f7zKqaE!H^CQa+TB zc}7)PChQW+rnAiV42v|ZY>Cmu2F{2pp8W%(OEig>tu^i4Z?M|=Q(6U4$<$QdglICn2gmBb1 zN~q)fHk6=-@+dnBX9^#27DDnP0&xdQrh!fRr;Mi{iVL0pcQ_r+a?9uzhgF!xn*q}wazvLUsMoGXgz5LPlicE8k? zs%zuI_F=B8erWn{XTbbQ)VcxTU%3$<{ zD{{8<75dEaa6|*#P>q&Ri9R5dN5M^A7lu-t)se~s#?Sy9^PR23XkYKM`<3D0Z=|8^ zXRU5eRc`Jn6wAeXGA&nu$<|3q+5`xxcO_yC|5bs129NjMj$7Cs1jlVc7F&)i3z zBN9Rr?zVt8yk+f8ua;e+QnD%4SN|T0+n(RyRbP1ZnS|QEYY{1Df&f3ce;`<35=Oa6X=+x^E{u zhJDL2svv`>xZrr|LANN3I{Zfah{n#GBA293=vDRJ^Ly!a{O=w6u{mZ%66JPWWB1D} zo+Ra!!@zxOUa*%;H+XRgf@h(r3Ty`tZ|&~3Nlz}NMQE#?(rPIeVge9QEJ~b@a3LFqH zCFqn82B^*XL|a7o1$ge%PyWGbSao3#^3&PN2YX z7T62G0aZ%k$wzF3IXjXB4A5|`Z6}hNUXdZik-+cYmSCcYyBgcI_3Lj6^J!e6W75tD z$+nMWPdg0Pv*cr=Sy`3XM1rp2FxkFuHA!$&Ok%Ipq5kOL6}NER1)S0?BZY<+h<72X}O?%9?307QA=ubT6;qtu~4NVl0=wH%ZejeRX8gdX3Lun z_s06$Rrb5CuXH+nB+~VoD>P?~v-P)rq2Fcrs5nfjOkzH1x*U6KtV;i~O{&7=@%{`9 zrXqD9E;HIy8gaVMxq8oV?9m#D>HNz!;1Zz3^A{lHFws1u>-zU>5P;Y4ZVcj+lf*PC zF6555XEZBphW<{+pV7zVOaBnpRmuBskYbDPyS00OaaYU7!NMImUzbrIqd`md4=t{o zlHrqiW3e_9sG25O(>q+CNFTDvmAiDr3ld>x3x=io|L-!6c>4 zfh%g{-ls}YgobyB_l6}8gd!D6Nt|>xBmot6OH(~my;!Rc1$~RF4G{9{1pL?ma+ifw zlxZ;BVk~v6!juQSVwtf<@m_n0r7213H!jSiN}gOWI~D|qNobCb3<8Bx`QvlFZV@It zD3ZJVAbs10O9yp9FY*wTYVa{`XhAe%xTwhqx1zP7E>SR|Jm)t?<2-8`xsiDrXdPdq@7wrq*-no5*D;;xRq0f-0ORforYGYNNb2Mn+nKnM47_ya>ojj z%-2ypf&wx!1r`vL#%zKZ!BUU}$BOGio#nePDR`yu~`| ze535{cnO3qG)+N?9s_z+Fshs?&BXH3#G96$&4o(|gemV1yY-;uKlxG>P9dgI1!_7Myxb?o4^Sb)uOW*ur3|G*t>wFxp1>!SzYdfRA0sy&BYVYH0` zmVXKz8#LmyO_^jkSX@wzOFp-LI#&QW9FEuTT~!URg4PW$EN9R@B*4=sQpT>eYj5kA}tdbzjgg>Z0q-ti+}0$pfvZSIp=GA)P@)w+ilgmxo2jIw+_O^s<#Ktwx0^MxlDTmL)5xtTe3nEe1aOtSdP28HsqM}qD zI2iY>NsdU`f{xbZ^`wsSGdu;5O*1J~J1Mj^4`v)6hEIOc%g^&@S$E02{WE0QHGMO~ zi95S@BXX_^E73H#a}C5LkC9^gHKE&{F+ELk^LKzsA8l60{gt9ENO*xP6_s1d1rKd; z;xOcNMG+A9Lp)?uHuZwzrQbinaWJB5%%7;{qF3fz?JtrognV{z5=yBiOu8d#K0Z+q zFHwnhYNWNwawM5XYcQ*#_&i>-rmkPW;lUsONF|^yAnU=Q2!f#L`ib zMV3^X0v7T*LI#Y^3}xpqDQNk^JgGtjFfkjDsFZoCo2zzAkpV%oC2AEW_Jw?3GMsfR ziHhW8$on+>-PQtW#Z$myzxtpg9rkJNi#@Glxtk!e?+YmcnYn6H_5C$z7`lnAouTK5 zu+)DmtT{v%F~z6q;t8@foIY&WXo-PCi2B0muxc#Uu}6%b<8^RF^;ntB0Dv6bdkeAGqtidMl7*cq75|SF(ivf)d4&; z%vtZir6EWpKRcxfNC{WLogZW5BQ|Q@BF00Z$jim-#*Q=;x2sdoH9srH3v*}lJ`5%s zUzj&0H=_OMw7;zWRw#CNbUvTd^h1islJ)ilWM(TRqiU{%&{Pztb}RF+!o<(l#Tj!A z%|)|HpcB=S2s&{3(zZS9r=o{F)gnw?y>qH9do*a>J?f}=ZJ#s_0#uB&%bi*(kF2XJ z@&YY7M~k5NHT3a)wYsLJ&{Y!2vnBk+=T90fSzt7TlNdVFt9|2=V@INB7}nC`hbter zJbJ>oIN$@eep1Kmzm=U#f>Q%jo!Vh(b*ykQk9_Za=^m_McmHx!NI{FTstACXn8QQZ z19N;W$)A(vR>`u!RHF{RGv3wqT27i`v!p&@MpL}t7xbpg3BZXh!K#=1H+Hs=OCf3SOz)oZCSKa4R zZ?!QQl9WB!NGBX8C?#C$ls6G_O>e@<)R4#kYOsCRcRLy6LOVtH`gO!gm`s~DQ#i$| zBPHBbzOc(tw09>;{mA=Dg_cNceBPVOMpkc{kK$?T3W?oVT)Eth9x<^Ta!#cm{S2=U zj}JcHdz-UA8XjkUjUl!!WLEQmM?>LGd<(6kR7Y|0ryoowSzVRfL{UKON=F(yQzO(& zF|J|!sj`v`u;Hrps?uX3m+;e=z`?RY0tBC~BX^+;Ao33?2~!1pSGb2w@G3lORk)>V z6=+lVj`g(0oB%4iIl|kRmAZ-U0l62G?fdIb<2uRY}`~ysd1MXRRrn>I+{FIxx#4L9hky`!1p3V6Aq$Q$i7Z z%gFV}JW#~DEL>n4VA;n~M#Gs?y4P6Jdx-oXWxJGf%X zjd`}D;dsB>o>b4kr+ZyTr{6d~u^s?H`)};*5WR@tR^bB#?w!f-_bIe7a;S3gH2YzDZF4 zq6IRA z2Fah$lff1bPt(%57$Q_v%QQMR^c6UgM};9>GBjPa{)R9W$${bB90{s;O-7F%5&o*i zkCaGxFq8Up8f>(65Ef|NoF$j&51HdGgzEs?yC*V{-Gp4r2wNwsi&FfAJ-Kc)#!H$T zQ|RGNOql2SkI|6fB?O8>S|A}rTb&T3MoWy>5HeJf^+*ir{QR;IOA&n0G^1_UiHG_m zw6;sAP9_G_l#Yw2-RD--b}8SSO{>EFmYOpNmPdE{r1Q`I)2Y0M-w*S@6t9uj2H$JH zy}Y@(yL+)4_dhxwo&JjsKKqBsyT!X|!(fl2yXJ>)Mn+!iWzIGY|2dsH(Mjkl4ko?$ zW_UwQoT{z0$Va|?)pR6gb@1O0kSOu>Km6x7HjGNLL^p6+cHGRCZB`gs^5+f|}e<@((RWu-X50j=> z!c`5&VbZYZm=KkYvU=#7gDQy`cNXGKCY7}I7y%NJ2Eu6GWY;~CO~3H7jW82Sy5fpM zl3>DA_b}4#RgR{TdOH@@svWX*_n7bgY5#~(bGeKf?F|zSNV^u{pxzdTiP`_BHhI0I zM*=9#Y4F+n{8eX`oqqqhLR1V{#L3I-=7zT$c#!xZGeEVTW6{~Q!SMXVKh3yI93y=~ zXYWO2jR@n(^IBx$YD`Enh_WI&=gks0-goh0x4AkoisVABI-T4!l(bVk-G2{cC~1&V z!~G%3$ZaP8ZRo!jB0Fm&hi(T_SU)VJ2~BpyDiwAps@|0SVIHQ*UaOK!f|FPgteQJ% zh7nPN#CrH4of7AZBatnRSw65eMc!&FZVsF|i^KA63^c>i3me41*aa~sL5bl#Vw>1X zWSywOH{gPpUZ+_A?|U|rU6-wkGAcD^8Hf{$D`{j(6o(TLfz8~>|DiDe09}v=F2PRG zB{nTB3Oy(=FGI?8=WcLF#`T2r5y+;4mjU4UGJTwzDm#4Hq}b79YLVI%CQ zu3EKf^;Bhljy4C>S|&DsjQdTy4}MZ;nVzhY#n;))x2}^@k1A zSz-0Ik^{X+D^W0^F+cDL^d!Raa`&?LG3GR&UG*}nUQmn1P+K+(G_7O#=mE1p>GR9VF7-kN0|*dJ!hWfk+8y04P0eaYt9@9BYk8&%g%S z4wicoAz(I9!4k`@T=Y(`iIEsv*VFyq2tFSCba9RDkfLzb;rvqt29qgr&3Ey=3lq;4 z6EahWLr!lfa*^;5(D;J<#PgB!%peJ}Ii{v60<$#obufmRgz2KiO)W{RTk&;>0R6ik zgKVLpaPqo|$PgIQI&a9X?*6)TFfK1AACYtg)|2CwBw#hgT&FzQFxe|^L%gRwPLQoFUhq#@t|CnG_{_e0cr(Hm~B-=fG;W1jwLT8alR98vs>JZ|kAxr$9 zw9V#5_F7FMuQ6V?7I48>P0xGS2mZI=Z&_$os~&i{Jap!N=RL@0{3}b7o_2?W_QNd5 z1=VRVJfWTf@W9$yQgYpVQE$DStv;57>lim-8VcnqKX$*YptpfoZsM?wIDo8epxjkG zqed%6!)<4jk{yQ*V=iKFt;*V~aRJI>L$tQ0_~lft$-d>LuqNRujm%?phOApxhJ*8x zDb^uLV$K@wnR(0gZ(j*qU&9kk=Q!f@A!=D#Th%v5U~F{*3ZISbVvvUD$^hifl54I( zXnXq@Wrx-#E$b_-9IGt`|7=fjg@qzV@Xwg@W+hZs&UNBgX`5}#cPVR2*chV@#LsJN z#PrFf62LMOA)?q?TE5@?_u=U`5~z8Gj9FcsGs5?!{M!`Yfm1ZF)zjq#UrkI{Z{1>= zqU%bVQJ!a8&t52hIhn%W(&0J?IiEQ>UgQp(XYCg!n=35KK09XD5tdvJqo&G&y3hAA z)AjaQ+v;l3El!H0$Ev0F!hp5;KcNRn4ok%QBujFL-G0nMJ5&g6qI4@U+-^LS7dV*r zlacmsJ<{s@(UF!c437}vt4Z4Q3I!n6^ABaPaCARJqB{UTLRc1j9p+dl#x7ks#|qW8 zAt2aWE{M+Y15t=D+;~!ta8zu@l)s_wtv+#Nxlj%}=y(FncX)43Zj7YoN`Mk(FiTpA zf;DGZFq83HgydsLmvY5n=^>c#0tX3`RSBZ~N^Y%C-oNB@BD=kAbsvldsvFh{nr8@X z9Vklgs8;Q4L(fZoe&6o2-bj86~wL&N{V2t6LC(Bz@+#N?r76`zb~{-*sj#_mfQnSD|| z;vUtm5oc$Bw-xZ1+A=P$(#3d$|3xF<#(kRC(abv+PL^A3dN0)RQ2Cl!FopWjQ0>?N zt6pnv;Pk5b#tFW`hRpT*N8>4MPLZ!V)(jnMD-^o@q;ww=0L@Tq6Pry+6bQ9>HB!LchcJ0_CRVx z@kNb@TxZxvK}sl#fNASFWU_U|W<-UeHxDQuSUE<0{I#C^ZFg%(Ez`Zna!+eVQN}J8 zLY!(@E_)!titH`idQTY45#3>LraP9qYAiwF*UAhfO43eI#22KmX55!g;(XkpxqU4pD0)Ys%qWRPl!#inS$Sp51_g`Z+m*C zT8yn6*)ApBg)4vcIRuB{{wQm~X}oyr@GeHkx}Qo=@C~gh*yeKd`Nmaj4Z$SjiLY&% z=sRq2pSt-0KI&S-O8S@@1Qf0tYk##@Kg1(g$yZ87Hr~P>v6(d3FRz1N?%f!6R+#QJ zCb6y`p&e6F(tN1G?2NWPs_2qDtfteN@`R@^swbM|Ya2{zY=oczoB192q%dJNv6RSe zWVrxx+Z?Cct}yAC0h&U+Topb7C+P9zTOdG<(E;v$K10DI%UF`Klt7)mFYI({b$)#J zZ`T8B->-)%{nmDm)CTJU(5d|uZyo6$Oi~uov0d%Dzh5WP3y4AJS;zwmq*%cv6DVj zQ6v>l6#HaQ2T5F+C+z>D=aqK*GHXp6sl&w-b8bdMG ziEdAH97$bZfixvCN}4pjXYVK61&A1nxS$jl#sPr`i5&uOi2@6{4Cs~N%B|t3UjMrKN9fKOHT}s=J6K;v&oPu1j z8x@DB>8 zA>kuM&T~<~I+W4|L~xI}4%Dz|aueO7f)Itxf?-_`2aja--fG7@Mij3ew&zn__&iyi zN{Ye_czW^vawgDJPwsYJ(YCaPBdW>HIx$OqQkJc>{!=e3+E(zKeR{wHMxPXRcg}}` zZ2dYNGSUvzc2_%KbH9!23Y`((z~_a2p(WA!I=L(Q+F-g?Awjiu3vnk7j&F5j^VyOf zEbSnGkO$n9N9V%2C1txs=+vVA7|W1{5pE(1R`&}HO~?p0Z|y%z@>`eOU(wGP!+Y0f zGxOOW%YWj$5>zu<^6p_dt{s2%p{H7Zx7)VvHmra)opr*!{R;nf679}3B>6$RZKD#2 zk0%b{Vp!N4D=Nf#4i}8RX1XYSCoPyyIwKzR3CZr4(z4jjf2hT|YA>3pP64IkP zidW+C3kDDPeXw@zn%d@-820>Oug!Y?W+=9lo-7X95Zy0er4!3jd9ds`_prr}53?jr zi}ZHI!P zU6>|2q{WR0p9!Rw;6XM$bv1TtBdmbrYKjnXq&6;x5H4i`+{2n{;o?I|#nq>L^D**x zMy>$w^8%U6;~X*&kpS8(OFhXY5ERu;>En^nC{<#=f5IUTG`DvH2Ut0eR!Fd+7Ek6} z?w1_gjUC7emPq=zL*3Nrb#Q)hnh-Tg=F6RgF7<>bNPww{PX}%G6+p&y>Ml$hJleG$ zu^02JFmElx_DvucY!@R1+Bk|%j$_7_X(R4$u8$GJ(VLoOo4x^ zd1kc6^W~o2GXJ%LA8kNhkC!s6lTuG~?6W8kqF42YZ^aCih9}ezFm4x6heUGcV%ZEW zutEa4PcB6PD?{NKmb=1F@R*Ck>VwdSSAu!5PZY1r#DlD_;^Y4j>R9{A_|H#;7-loV zmp;-7E!pvOKZhqQHK-r;(JtcM!b_>!_+$FOX4p$b zN`!=3TCbwbW{mx&Ao8neb&p~-CCHJBM-&glbOdRB7Rqj@|Ht3dgWhKC0fK>J$ya8$ zn?qA2m}PSVPTTk-pD<2`38=^23BPhg;Uu_0*SgH%pYLxt7|GiT4VyoNacx(@kHQ(!QW$AP z4=TO!e87#Kd!gvQxpX&|hAh@mpS0dh_S0SWxGRl-<$+?3@V;Yyci5@?dxW`^n8>KbsQYSyItY0~)wY*s7gaec{y3kW*hF);r^QPIL z^ycFUkNCQXS%%CuSmDV{OnAmyv`McP0P2iY^#)HZR7JAy`6tSHM(#LjRTJD!qe-ES zaJ%LisLHLs4R$OAoxNa@Zkk}VH-wG;ZYj(9AdBtz=N9Y=R6R;@2#!8(6}dzyDTqn@ zd${cW!!fY&?6X1}`mkp_)1|)nvXF?FMaW2)7ly=coNr?PQrny-i0kUUBYW+%v$g7* zfz`ZGW*Mm-w-<>5bO3|j=}RJ8*9jMwZLnkkOw|jQ{}_cK76#ducn+?pd0cJHT!&x; z62DC-<_U=AZaXW8ADv?pp9?_vq2||&PE^{a{Abr*0WhF2bYGDsp z_aE8;E6LeEUU)CB77O5{KKcvbS7r;~S+hld;w13?>k)WqPXVKg;MSAx>a)P8@s6B0 zxU|i%oH&NUE%mbCv48uoW6_VD&pOb-FnPw#=Ufvf&(PL-YIPWWxgTWhF<7jIa0-6ZXAHNrI#$N7n9T{wv}x zNyr~=W#cqCtrh@*F2@FASWeK&Y{ENo+vALjW1VgI4&4 zNH09M-xE(<-MDH?<_^t>E?Fj9#f#;|)yWN{JgEP2d+d2B=Iw>l^?S>oN%MBfHFlX; zsFjTGrBE%vt{_@s8Qp9X5WVR^Vz-q!YRiHF2YAq+Orws8rHT`CwXx;N zx=t{;a&6n-p=O1e>u7IsK=0G&xJmxhh+BJL&F1GS{~lI@`R80bnGNZ{R3b5Rl^Scp zXBUf!5ZV}pghJ0k++fOZ5tTZdqT(im_pQb#(2(4!2yg)TujueGK6O~uPA?)TC$^4= z57^~r%~AoDSh1$;&SX3FW#C+7xS@G~gfeVDSkG0vndQwvAV+sEi2XQv;Sd+Bo|!?~ zeyhzp2RE~&XL?>ul4x0_KEB~@VI2P^3Xp4W0rw)Px9brc{46)kNrCaSzH_n~b(#rh z1_H%hs=PMX9Y?X43RZ>@7f0|om-DP(X2GOg%o+j!GBfbhd5rQ?sEY#NIih}j=xA4Q zazdMK!mroE)oEg4FJI>GIi4T;cJ@#4`}@oXN3-vaI^XHa?7~ks_xI$vAC})UH#4H? zhiX;tfrh+s^X@f3Rb7Hbz&tHzsrkXL=Ouo)_A1%T5N1`c`6O#J|I|3xWL5r9d-1ZV z=$c_S{wRHn20F~qSQ%Z^)Q@yXHfplPe zV1i1vNr!KY^#pKcJ)HOJ65mgDnTrc3IvorD6IkgqzF%wE9{%v%L>gLX(Nuz3xUl)r z0tqlRderH8+Kg78{kkE_nI)7^l}`9X#c|-Vw1%P(w(NLW@JF&4200;0ugSV?NxTJ; zT4Y1Xh2}-E@HqRW>w~&pvYqCtLB|!PtFlw^bu) zV%^fzKhyR=K!#{5ZfUO2*y#z6n6!}S%1cpO-9H8-AuG$GNnd1k>Djc!e2shxtL^EL z8GCMcJCnmGv|B>?e)qTk57+h(-3fqn3p}=M+qP}nX2(g#wryMg*y-4|ZQIT} z-@G%MSyt_9Q@?woE$`f-c?)hjm@h%^xS5RIvsYzlJc~v=_(_Y%Rz>9RhoAf0Vs(k1 z$;d&`Krb3$5df$kLM{4+q)&x)^X)r)QL>)}WVQff$B>Qwf>s?(+~?aN&(~48)DV7V zZ`E#*U!`a==tV%t-=RLI<>MldX%d@E3B%Rvt@;3QfekKv)oCjvQ!z!kRpZJRAtsUI znc*6hyR~gq{btizk5fI}G$1e;0ddEey$?O#&yUUYBRcs8r9>0hY93LS96hAY6+O?WeltcKE5_tbZl;2LS4^#k8&aS_&wy)fJ&y!Yj0p zbdz#tGnZ?Imf*0t9C!whTfTNO=+r+mu;Vb}sa?=hCx`Qf=(BGgW-@20B$xCcc&1H$ zAExVV>bfy$0K7)Cc%9c3pU6ex+d7U{SnPNqx#R-iEF4n%yDiTwh<3G+kMZFa^8#oeV{S z*qsarO0VMl$Q`*S3oD6UPcKxAC!vBa6i&JYp~oXgn8h6IR~FiV%8yPOS(cF1Xb|r1%laWoGK2 zUH;=N630rzSl-zDr<*zw`6SEwGIK>&Q5`ykDz?G!7vu{j%`Tl1T>@4!1fW@#*N6s{ z=(98$e}mEvf>PrBm{ti*d#k&tNTFMz;K*uR3?!%<hYkIl{BQC6tUI3-dzp&}1g7DOscCZ; zaZ!P>huu2C?U{#x(nJwU#+#{WL|Byp^921BNb0A zG@3GZ-6G;83{I+7{)JFviWH-P{tFChK~e=#of`7GMiuuE7Rw}|XP&1)FCatYwn2E| z;`=qdYxGcH)Zsi>`PgKAw(t9J&R9MAeRZrGPq;37F5#Q<1QA+0HJ#!-LG9%^hNB;W zl)mNdYr61HC@mDP=O~QKS`Bv4?}`o zNR}ieo+w5YO=e(B28mk4XeIqkJ$neJPv*Dj zeFWewvi-+Ux}nV!YPY`@*`UjOxO1Z&CH<<>OOq_8*+GsY5VI@Ca|du%ST<`%edC^oe||O6mi3VY`*tz7 zIjBmDEiYIfg08?_W*CJ<)%GE0Gn-R1%q_1B$tulM5P#p_g}Tj5ze2G+eI#$Dgp^d9 zY3(WMfGsW7Gp#kOw5uB#;JGe&F)!nDJehwX%l1|~!@N|BC|?ZW%g5kySeGh{?gnue z)dk8_Yd9I`tidKCp1jxhqpm7cJ@Gu(%&M9Ti)d#nCktlcS4oiLYn>82(C^|mo{%MD$^`VhVoMA{5;1@kAQMBd}o&r(O60E&aQO^?^AZavqK0VOYd#8*rPvlHPH!7lgDQ; zfXtTC>XHH+S{5#$h};^Uk!^NO9@D_4rT3r564~T zk?tQ}fusVG*XL}@(d8y%ouxhz- z6|hqtw!9capAD^Vvo^L<@yZZ3ru(}Rp} z+uVH{)UzL)e@8_+wjxC0>?Wk4Z_7S7e^})nV~@sG!I-u({^!gx6PerRl=P<0TTAQW z-9N65a04fOlxMsM33LZ6@73>GQn|xgA4+ZRa+L?S2*k5isMT0{lQ=p&0mjJazbKMG zFjv#RRV{M&Tnv~;n@(~JNLY#{-1q=PKBsWC=oto4P#q_ccg%tx(Z$gXb3UqU-=}tpy6K@99j&5A?D~%$@VzL~>s~YU zdbY8D(p}$CR2qT^`f{Eu>0Jg46CPK8%{x@HVP0M>9|h;%6)hisSq;Aq5o^EbAHNnF z_0khpppIA4{=aAKF~(FN1!y+AT|^AF4d^5Ew3*mL0K?&OFbYb}d7 zQMBVtxFz(y2dGZNKe!A-_FMVIXe`m?t;HpULb@(}79jPL08-227_=n7 zPNUa<7q6d`Y}poOMC&D%unq<{TLeezg~ZexBuB~f3ImKS{2`)(F$S`&uts5)U1y@B zQrrA)1h{Hq+evA;Cd5;(^_m_P?WTSb3^7$B_*A9_f`NY%!+csJewmmIkE4N>GlhQk z@7LBp#FGgryIeqiUZ?*~=;Zr^^s*A}r?l7$s;+d})~iAdish9dL3sF?Hd}-IwB?Qe z-oGbxjQ`$yKlsb_{oXh!_Wgp^$@l%f$@Ts2fP+jPLAek8+U!!YO%COfsq0%|4%G{< zbokTTc|?2~OIMq@j7L4O7h8`+zXhc9zVx#7mveWYdgj`Cm(29O64>4;?JV=h z{zf0kKYbf^N~h9cdb(VQQKwl@!~7`Z96_5IsilsFm@F2@jQ4VS^$*T5ptt(JvZt=! z5JbKt`+x=#-yG^nQC^$LJJ{E_e)a^vMp2q@ zWnngKg+5VUATHzFv!h0~K4-GM*JXS*EsB3F9C((DcHZCePOo{cJoGTlKiP)uHu~%8 z+vz}bB(>2`GSv*E%$&RI_M_=R zRoCJKOs&YOgm09R;tE^q*=AonL&az=-ij@+#l?!rx9E4eVt&7gA%J z_wDFOH%@Md=_m&UWP4%daQfC86Vi$Wk?%~4e?NzT00nEg7?05{lpG@c9=jW|S<`La z!xZy!6)x-i*9jp_)42x(DTTslWY!K@>Lx86WR84{o}N3Re$`S?@5kfiVmW(qlCanF z@%LAm@_RYSGqi8qj6V}V*y@)&zOlYa$)fEb+kgYk5_8M>0-M!@W|d%Xg;l#(2qQhd zLa;U1E$kPtY6lYu4-}|p(cGSS4%_y^Oz|u<*5$=1o>74iV%S*N1(h`1jvW<~n@U<1 zar-;)89?R&*W1Zv`1GFoH&B3PveW1BVlbOF+C~14t%)w`=Edw8=WJ^kN}X!t>Y$}) z+1ep>d!1+5+TqK)+`;GnjV~S~if?iaqaW{bR*Px}?0or^Lzkz+<`jYKUtbE`17!6Z zvt;$15WLU;RL?9hjsILQ%{@=52fjxcN?mdr<6oP6e94?H-(E_GU9|)BBE=fG!eG^L zqLC-i$+IN$ijU#YeuIvv`m{BTdMsLjK(Gn>3~S2_iY>NnbhcAy>dcGYY3s*7#0W|( za6GHt0;FuFr60YK8XVQK(Q1nGi)C@Uql!dWZUI>KxC&cHgD;wjtUAov{MU4Pw}!uE zny->yE&rI5@f3@dfL9b-R8-;W%#bqiJvvm&5n!h@g8pcCpqMM9sp*R-XK2y>Rdm)N z$29`V{TY56oR^czEt_M`&C_*d9#N* zlI8Et3)S3dD0R^hr`#F^>(__=tceM?8mC&qW#MviB)X)_8Dj@p=Y#4;PA`6zy1K8) zp=R&AZ)xlDz)8b-8|2VGJ#Jkw*>=(5-oIMV5+N<(g3KEFLXt+X5v}z*wH2%6op`%^ zas$5e$?(9>u3lEFF{JE`dY`uumGP2Ad!vbaAl9o_;-&%l6vm+J6P_P1BjJEUtb_j~ zp^D^&-Ci0N3GBoEknIo+sLUxwQ@^VcN>nEsxscgvkms{gbbFG0D4)>kzcn=sO*wLb5mu~q>$HxUCmMd3~qWO5ZFCM;;? zrUjwc>j_!(u}B!_eJoNY%W2lN=VFwF=G@rW+1MWn#yE}-)$#bNh|(FVm*+-(L64zQ z+LmH@u?9Y)aP%PET1s9c#%*%XFwIF(i{(u$tFg_jA#_1iqa{2jd4R$|q_ojjH8eA7 z<2aO3i(?Y7s*ke>Y~T$iZuzRGVa!X%0p!uq1vUwTMxL7MX6^F3fmgd{!hVU+QSK3$ z<+5ovk<%lTe7*iu=LhyB;~;JfyJOpKWUAZ{!d_A(LkUN-?Lq3&Nh+djU{_6InO-+D zD?L&bmFis9l$-h&nZsnOfsRxysZ&au8dW*$wnRN1E;{l14e@)h={?`?YcyH1N){tr z(G*};KV~{c;8Vb5BHJxXaS6A??Hk)ch65?UL|(yWG;f$J2!+-oMXi>h8nxR}R+`r= z;Da;5)ubMmSrb3i*(rkD$%Zcv${sz;C}~uD+0pT`X-Sxs?1iK>x>@;;wzEv{3!M0WA>8qbYF}VZT&%8HE5Y zYnC>LD;khW56fTgG>7V3?$iH1N?%fkSCsCG2^JG}&er|Dmw~#k==FkUjd}vMT(bVK z^}x}sbD(-eR~RmjaNZM`Y_wZHXB}OHIk3*CG;W>ul2C~0DyygkCwak=I1#HzCnZt8 zEbBL&eomT3)*5y>((Tl@>#&#&6&`!r^~baCHupgUNX@z!NRwU>w&C)c#oKqWeCG5y z97?lH=ub6hVu9^|*&1Z?P}~EZK~Kf_uICQay?F>CbnulW{L4Z!=jA*nS~AEiEkD+Zw(g?4V2cbw)=)cExbK8bG^i@Q?fNGAQ>*;E zA21cOxTh<&P25*^5T@LIix-hm{fQrH>OEtxP<-bz`{DW{puT;siImqEQB zdPGgSh+&U5$HDX`u)rg@hxyCOc?NSyLu zr?0&=-h!@8akB)Dd%k?_Mdz$D$DYlZJFN@y?z;Ai(+(Dvs$hA&%4bU$BeckChjVCq z+ss{AN>5DP34OjRf2oI(NL#9iNohacENgj~dao52slE$c?`w?j^tWzE(ZHOSb`Y-^ z;--DkMjx6y^3EqaPyyP~+un?%euQpAnNo{3e48?+)C-EC^?_TTVU|!Euh9=muYYV=%&Kp?a=e8-|X-n~^=WX=dkPV&t{p8eL(Es~! zdDg+NhwIp%`+L$*QbB47{U6nO6e8BcYjt4%r5Wt)W-#M=)?`$w$|`kTdvzkiWOrUn{m34D*?;MZ6(N^fRce4vb61~nl&oeA|(Eg_gt6J4^0Z!T`ei*LW;>=qMsikj; z4Wv*sb%)IGPv(afuK;1e3B&+5@1)aW(ehrW5X$lh(RgMl(5)?z&OCm#xXEIi(mF<@^1sr$kz7VEFVg^Frqeu}Wt|sJ zGO@w0OrMJ>mSO?F1U#Q>^hkkOh%}5hjP5YF=3f54+WH6W7eo(Jc`=ynhn?G5MVqxdL~?(=}srvjb=ChPusauZ^E9D*y#>gCiO z+EoM9w6luUWtYj&L9q0j)JDcwcSSrrZV_SIRd`q@%kM}tO`6@yJYqd%9k(<~njMLA zNSdX1zfwMKVxA}6w)q)Bx6hPO_Z|&Fw>5yeKU@RimwIR* zvK)?E+Efz-Qpcs>h1VoEtm_BUVWBcS76@P>OY8)pA7B}gNij@kGsXlQ4J2Q#!S1aJ z)Gqyrg=Pw4Uq+^BMEMz8KTM^HFVum%%xG|#m=8@pP=;*7Br6M4UtJ_=IsDV$Sz%uv zwnl}tfA>#XC>lX^b)cAG&dsz#QLR3Hi)7GYri5t< zOYE0$8WgzMRZl%fZ{dN+9Ecgs_}lIQ;ofBAE^2$OXmdXdvL|L>t~NpB>;KrxxZ}g+ z|9bMhysrD8L7V0ki}FXzkkfm6&fnac$)OxV4jkr;I6HM z>r`Q&*#KJQ`WZ_kMysl6Db`!arGxv>qwDa)Og0uV&k0>}k1ulsTArDz zekXgZ!@vL1O{0FpEu((j1ysKTGY@`8;~b)X^ACOr>3?=H_DGIfizyudNHdi9o_0+7 zsG(PZ(g;xQ#gFWeS=*m$6RTir`oX)!DKL5U?pQN6Wr7i1(S(1*J6{{$RjJcrk8BTt z;cM=mg%B^eDy*j=x_UGwz|%_@nBI7Xi%-~J+3z zhCj*+pSz61gz4AAcASrb6jjKl$eK;gQP;>&C-fWjwm?#-io_c~3&&5N@{8D*w=L47 z>1h)=>y0b2K*0^HB2JUYNoKVZ5ooS9c7sPt$#-(S-Q+!pX(hs)NhwlIRk$fq2mL5g z2jAsrAT!nC8VM53!k~FDBAe()vwiUF8~# z2QlW27vg~D<6=<8{S=|4N)64mY=+luWu-5V)352MXnv6nPmSAwf(y=FK`((#goenbGSkq03y&|T7N;zSnk`S=6r@ER z!xLEg%D_$0Yp7Zay}i=n|5)Gu5pw;D@>cu$*w}=ve?CN~8tjAP*vlDdZlmg~k;<0{ z>%30qWWxXJc8{I48SnmC2cP`ZQyrsB9MN!#7GXCI`O^G+zM)35Nj1+uv((aiR2cLwF%3j7 zw`DmM=h#cu-=?45k86KDQcTWAO+Y8>Q8_^7YM{$2HvcaU_L&M}iT?bsrsjY&GGdn! zlK*h^Hm$PSl|V+ru5b1qu{4=R^m9Jbwt|zxUFe#sBt~Ee&-Pk%k_SCBjf7+~_2Fm= z)a?;d%}dj_@lYx6V64kS{qZ_kW!?t_ua*ep>s*UI6ZRY4Tell%nh)KW z3*Y_$0!UYnfd4ku?oykf{y<|Q=3AQCa#pz+G=+p@w#>L7^%}ldIQw(jROIR)n=c#a zGJY-(&Ns;CZX?U7R;2Uzv+vi-pi>mn%JFtrSN81bW8B$+iU$Bad>*Zcq(e$xP3GT# z%#j0cnixPY$fsfPQJ3nwoB5n0)QWARSK&201eb{srsFQfYd>l0o&?>U;VsKCby+D8 z#+TQiANU^JfmQj8vwI3`rSMEQksc|=eG6tKh`9*SeVg8fCIJMN%+!li1G#WfBHDhdQ4Z6|zeA_T2ah1DXP?8VWEn%*? zyDMe9URe4gx*k$BLfCr|ecciFW1NOQXTQK>&jp%-x41T@2;?55dsd0-vpJR7i&tOONV>awXVUm${LiYB9K{q~IzzA<6m&D^59&M*7tSsCR$#=S zQx77|A^|v4a1|>khy`(!Cv{GEcN##naC5I|t>e&pAIy(3a}c~qGP}o{ZR2SpsjeS>tK-w-#92dm z%tJKG+`v(TJZe{X)xdL2>j(ki>4Ik+`RNLkxlvJUmdtxH`r}8Ji-|eTOv!4uPf}~^ zvuD8u^n$%|#r6s$)xNTihoou8m`x7phJTnQJ4Q)mfdsvyc>HH<=|@Oh+t#0g;)96- z=dWY131Ot=8%7L}9*1tY2C%E!>yvpt>x=*|Vz3!n&r*&3zFOk>$Bo6^#i60o)t$`N zTHW8xoweoB-h(w;O4&`^e+Pcs4WPUqEh7YXLo#V(B)bMFCdX4STciyiq`7HkTD8m* zt~H{m|LU1DVi8I}6u7U9n}x~PHlegVS4|z{Kk~I5WB2L;cSBJt`qCh7Veru?R3wfh zmb9x@K-875J%e9e0YT;R;+eWadjGa%5_pw&=!gW79)Figg2n*4I(8O3JSQuCn8~JKhZA^SVtVk{`LRy9V#&wK_n1dnV<{FKVW}XwbX^ ziJXwY-JqOZQZ1?!&gM90nrKEFb0l3}d##^Z2^OAXUHmS27Mn6B%$7HxQM@`MFKoQ- zW6~p9MhNzGpo-DE=#B+TTBkCEn-2YeDNv!D?*=BV-4~(@8}0ReGgQ){JoX;y`bk;c zzu;mz20^^IOT;Y$_T+W6`ZGptZbx#-i*`R=fFZ8DMQ6z#0sreJdR*RpXv3C)eQJn= z0$O6rca+$07CqM9^2Ae!M;s6@&x;kXZ4*aHXT}h8h9!XYxXx=<5fbT;`YxV+)BV&o zrz94o#*COPO|LoYfW5YGyY0hr*fmq9E|oda_{9=jtSpyqZw1a-tTpUqgfTrcnjD2Y zOQl|*nZ5&2543^Mr1(~Qu<9blh{0PMdPz2HB8gcyPm=stsP%CPBQv(w`2#wznVad! zgq1#7wiWV3@ecot1p>hQkhSnAEERlmg9cLmB3rL>ozO0LF5+A!6|2)%s$IE_59=yCOyVYZVH zM1P|m1t03-Hl%EW@H{zp-I;Uih>E%e zW_cwMjbf7nmexWHUYct79$mK-#>eJONuXZ82EUZ$`YwcEmADb@&Sf!hXAt;8(aw`x zLobaLi^tqh^FYl|$=Lc=&38~q6y$Mzs*ky$e~q@sNuybf>6KQCv}^I-0#xov7P1FS z68_eDHD7M1f?0<%dRxiv29Te-JazY?F6f#tJ6KH`S#3zgSVF%tC*|07 zh`p+Xd@7zl9*YH+6Rf0dG>@H_1S@XF@39horz;7P#1v-$UxdRyfl~#d?VFU1K_521 zMk6=cNfDhDVTy8$1@*uENDWs#>`W$h$}KbL=#DTAMRD3(1E$wqhmwQS{TlLNjaOxw zh;lR;j`%|}*j7tE4yLDa8QeF83eW&mkjo!d@_L?p>cCVvXn^S`CwaXBG$1~`j|U1D zbz-jT%I6q(qT#4DP?`WP1(mcp7JTBq?*Q(}8A?e36T`A#J~+U+vz|+|=w7QJhk$NI zDA_d78LyrTls+DK0y|(Bo8S_wActQohZx?5PDufeZf0(evLGk0P{C`Y#PUioId9*G zf+S{{L2dd3g=kR0P}d(Z1VI){MehJOU8b(0oXvqp?(Aiz~f&V5UwsCdJgrRW%!TCZmf%a&G(;cf=e&g=_1a?Fep}^9! z(^QS3($q;CSfY~UA0ZG|#ct3AWce`uq9pWD6`to*!@ifMjF#8#c`tCj*G!gO*d zEkF7`55vrS9)uqLz=t1?%cQvCKc(*L%Ew?`HU4Z1w9jjCDnvb}>_m1}FkBvk*KjaL z{&IvSzhPIF;w%J+$c$IjUMvi+$)qvUghyaTI%)*#J!g%Us1Wqd-})h*Kc`pR92~RI zt`8>M56NgJs`wBM%5F6#>2XqjZQq&(dZ9oP~`FNy&ck01u?RSbew+C=@<$(CfD zT}vLpoh4xN>FmFTcpQJ<+JUp?4;vPEV~824Q1Z_x6wr-*MxC+t2Vdyw&B+uD(!VHQ z(oVNjPopvZIcVMp&XN{IUcZ9aF7s1O-*w{UfdEjjti(iEU5xl}jo1+IA?naLe0y<` zFr)?#1eudE5v(bEJ2|y5P%4@KqB;cjM1%mSch zVXY8A`QeO6l!%)YdL|dfLXci8#q70DiPjI#jBk*9S7Sk+spH@cly4`{`I>I^Y_rEf zNWuTrDFD5J*G7VOzuY4V&g!l;C&$CS?T?=uD)@8!z73wAkB{&6dcHm{4`=(|E$06k zEWN}3T%IQPHgWz&{5g7hV*CvFb8>!$)m3p)lyf-j34O0~sE)HhG9uB! zl&jwug~Q%J#QCeAbQ+C0yHDolTjuxdbN>|&`;bZ{O>v`T8kbg^VIWkh1j2a)Y7{*C z3&z*lpJ)cIZ&8M~zX~uSSM+LfY3?ymlO{2fr~P}!{71ug=Wc@?!KCN4Tr~4Gpa3ab znD6ABR0?j8>9uaf5_aWc62FI8MEWn$3*w{YcVF|<;8m_f*tm!s} z!C7v6!(A&P%H|8iO%rq}X^(-@WRka_Hj1JmH}pN28-9V- zZG?Lz9G(rfqk>1K_wxORa>T}pP(}~?>};T7Av>|DEv~@nOposA3wOY^5R6c}q3EOM z__}bTUlWUUc5+2%q*QySvyXI3|4b-4BYUz6BSZC(SbU?oq!7KFqKTrN9cSOwG0N+( zWG*xree_9@w-tqrm`!=gMyI{7EqpR# z(}Fsd?A?K@fZLVRpP(uW@A!L=va^pR@&PPMQke)1eDp^C(>y!GQnuNnU*x+QvgU$u zJUK3+Xm}~e{#XKasjTAF8s@_k|hP3)?k+`$_~^b4ej%MXtibTS6MCwu zJ%Pa!xu9+5(U49yeP+xVTEs!j&RN z?V57%_NGy(=TAMb(c}wO0W!O)xs~&;%LSw3mpxYju=XHGS*ZmE86k zZ<_hVFJf`4Gc%h*4};h^5$F!q1Ia+eJ@Zr+)`Lh+VbRAWuX5Y>*y{8gu6&d8L$% z5Ea6Z(OP`bAaym>tl8vaZCJ-2vt+*>juj{cM{fF^mw(M48M0g(;cGrt;ryG*xzn(T zGa1B?2k%inMge>Qo8KpXvaqF-{^Fh|aaC=EE_6EOkR`2?af-evhBd?sxy@F9O(hom z3r#z#Lb69)20ZojfAMjFHEv$;AUurN>vzv1Z0U3K@*bgu#O65F1*QvNH8$%}@kHJM zs6WWL&C_v$ttk=8T~2UTay*ASZ9j8&H>kK&0$=KEzvTIrVh*p|R6T9KuIH1wFs z3@tXMAal${0%?eKeK+cwH5x?I(}ccXyki)O$Wx{}Y`hMnQ~6?XHt^n5_8xphiZ;nIgy2f1k z`rsGuQD89E4<*#RY%g3Jr_(ZqV2%n-msbs*Gr=o@93rmnT?}dL`$pFRndkhObGrmt zq)3PYc_wmQ&*}l^PcCCp1lLjCV<@D&v1E3*Wmc#O)b%1r z(Slzw$oa&?Ie|p^BmuAl#0$!vX&^EsuqVY|Z1=u9Xd%$&e2daiC_N3XQ}|YQZ*|EcB~M_su(Lf>?^|2s<0v_$Kj)C9B#mh z`#x2*Mni<F?Ps(}Ml``x(drhg|Jq-Lk;rCAZ!>%F#ekm7M{WfvkzMDNxOp zkjm#FoC~k*8OMBpmc|nd{^eI8O>3Gj+dYCj{nyEsNaf%mr-Z!M+?;Y>G=;~_HRrTw zysd27@pV!lEus|{+lrgZm!A1gL1zN=X4P0LLz~D*Y$k?x8)OVS&+1soit%5|E&c$z z05JaK6VH{<sL)u%pEJe>K()KQ^p+bpA{6iCmQ@_rHL8WJ5 z-ZVY!(=IIX5xK_1udDNT`XSTYmoF~p*FhS?F(ab?`EV`l@DyTp3##r~nAob=bC)+U z&iwCnI6Rj=ztmq5^q=9}xl~_GUty zciBz1&At?_&mW}mkh#|r9|;+l`KGue*Fl*}yuX9*r&3>rr?B%@Jh^tBcEk(@KEGAe z<7=>RqPtcYwev^D^MsNGEeF29QpM{-UmFaWq4y-rTbfaIZz!K1|+rZL=vu_#4R_3vb3c~aeJvm8g4N#p^k;(6MvaS%9` z52hy$Y7L4o^jBfcwlDiVpqUn#GusSgK(MQ$>%qPX10tZ}oN90dZO0y<<7EI~5?gbZ z!(pS)C{Zdx|WNbENx$c}9ZVKh+-#G{)X2v_kG+p0B;=A^l( z8u7{>PZHFIWpC)D5$h)Mz_i`@T!s`IF|o)?m_u@KG=5mcxSkQ zN~CRHk;8(&;mhE^n$sK3pYk5wRs*MLFw5c_HXdIwbAXKqukQY%gSEY*oQo885qLo` zTI6~WIo<)dPK~Z2UGY>|f>y$y!Vc1DhAcqB|8LGiilD3I>qM%|(qV|{2N=Y4yd=w3(Jh>*T;+MtmC3R?(L5o!E5ddjpvH>J8Kju@TN7x1p}Cg%4Vqn zjv5c3Nua`0y?mLJ1VPY@TY7Po11P{^AKj>$Lrc$wBKQIVh zgmW}NALv$2h>ll@K)oYDzQ8G(L(Cp*1$D?)@$-=j`4rM0_oyX;;?7SQLubg;v!qvh zPL(sHi#ekuZ*f?*dIt|^EA3--5o8k)U=wF=siLNu5<=#_Fc|Q7Q_GzUvzGsc14naliQp=wu8;>N^0~oB)pEkyuK@**BHyvXGY&$?EUTx1xL~rVFjAbzb#W0T zF$@UFuLjpar035l`$b@3HaJ?e?F*!@IieCJ*J#8b0vu>wSD>RguD4ufCD8^Z(PE-7 zgaEzB%uOn-LQ(d7<$tUgGE%`{;&7xW5>|vnR2EUfP=x|qN;%Z)kc4bdaA@;aOs;dm zLzSaxWr&DnIFKN0I6xJkx2Tq}U#b26B#gU(1%i-3KOyK@^G1TkYQ2d{fz&(K0IV2tLNbCe*plyfGCQ5sfeEsV#l3PirS(L8d!0ncJbc&9fw0XWg34Hkoai@C)*~&RoaQ!Yo zUkX~&IN;r;0d68@SH$(a#Ta&Ey>Y=Ckp=XKVe6jFbc*eL_nvv1N#s%C-B;qim~xR2 zQ9u6n)+ub2Hd*Hu%!({Kp}Y35L2rM~iWe$p`Lk7^q3gFHq(IMott=6gy?Ot=gN{&Z z!?l?~AGXQGg|CQq7M7|Z2?R3#wJhjNiek|?KNG?oF%petQ$5g-8s>w0-5={XggPcZ3}O#6xhDISF5eUNpzJjd?Not% zfM2}tBIckW`jORWma@)Fa>(Mf%z^t?RP;Jhg=_AihU1;x`l-*noAzh;7Fkfdhs>oI zDuLUX6MqmT&tFyy;|SVQeNskCSj=i4K+Zc=x)y`7Qj=IIK+xb_wV_K!;-7VU0`0o# z@g+b2zQMn_ut0`IKWt}8l}QRBg-WD`ISm!m$~DG)J}1fVV>RA>pe2I?l+M(53c*#Q zlIV6)q2h#jWXkN(RvaMXYSf{1?y^##IY8~Ptk;j|m@pe1tpbpx2zG_v+Z6hMsD!u; zlwUU!mjwz>`l6h+IY6`C0m_+b!=_&|G0HJi&ZD+~FOFa$Su<7Jc!cTE)W1g5n_rlF zCPAllF_?p6gT;Y?CndZRSM6qGD{{+M~8vY1lvLuV_)%2dT8{X(cyO zVM^nGJ}*J4)VUOjzGkawS`dW4(4pdqwh8Og$)aodTv*eL75tpk_qIX=e{r%p83;QA`$Y{5#vW|ncn}rQ6lD0})!s3tk zj?>qrjfN(s>lpH(PMg7NQhJaqP~Q#;D5zOj~+fn%wI5EsvZ`}k zqK-s>Kt9B)bKQdlVQ)2CrVR|^BZ>9Fp$sf~Lgz6q7MFLE1c{A9V^>da;PGg#unrl& zvloxfs$s&maxJNN$Fh!(C$-61&J7vbEl9AM-&XLeEwgR#5jF&LN@-_a6TLu?mq~IP z74x;B6&^}_aU##4sTjJXvbX|3{#%is|JI;w=>#Mrj3G->Owi*PC zGwaAP&{l~Rn*Mf5ytDCEvtu1CAdGnU1@Ca=R@9lzbc&b~0v_xtEDkbU<6|Z+iqK^L z4x8T5dZ}LfZ1>q$9T?M7p_?N1YwGO{<+AWo(O+@ZA0%03JZ$ zzX)SsLEXSA`;M%^r=svHBFUbHY{46(DF&p<3KE=$V3zg&wh<^0}+fhQe@n2s2n5j z!TWtaZ!B;4N0(6kOJrtvU-0~~qU-%da3^upl9A(M@T~EiuG#T1z~nQUVtTXw<KOEwY^^xxBDSHk*pVL+kTm@#P;E&wywZ>~6?%Wb8z|sQ&#+$MOt2 z3#tt*#DG}Q4Xd!LvUTg%2?M5qjt+7K?GedSme4^PmvQ9IN8 z8E2E^>w>R%LKP(f&ypC7!Ab+TBtq;kwMZ2d`?VHC=rK)|x^zhm?Tm z0$aL+*0sdpXjPQxSH}uoXw=tngt{|*s-f<1qA4N3#o!bi?W!Y|@uVUrY_VS0ALubF zT_YSRw85yT7W&UK<{#Lo>9J3Zd33a&`Iouh^+nH!H2(;P1ypgp{zN;w7_&@)U`_qkUOX7z7v#HsTI|9uzg8 z36wjmHY8`cC=oTUVOB6A;YJxcuP{m`5qXZac`HF~7~7&n5*N6EEXeDu;@b>%@cD~p zuIs>aCdp3JCCNmwR{A*7GsQc~8gzRkW1XavnW7YqCZDM8$p0kIH;xDtEAZmk&n8~t zbl1jCoZeYKr`FFK>*w71S=zssr++ye4zUQ_t2L`$fy8&NryOy||0eE;SXHdR0?oQy zJLr||x$Vf2!@K+0ugMk9nWzUei@hazr-7Z(r!2@FdyR7!}x&f2M7gMp^3UGZnQa7!UFHkfCo zR_>COx2(L%B%ZyP)qADSidNmO-d-x;#oWp9=czU-1}T2a~HaIv$CbKp2=>3d~t zMHsiJ-nvfcS3G0aE-;ttgq0P(0TloGhV9^=c4fLowQy0*piG<4NJxrTs+|>BY;~b< zUhOdG7w+jTEBOjypS(*2eYQ<`(DCB@6?om>(LfcX?o`Y-M%jbt3R+RH(@Pea5vENU zkL7iVf~)PCWHgL!B{VASqn0f?pcK%tt@4=CGNWNUltOgN+1*oJ_GYexpSi9}_7t)r zp8i`*Qi*!5zJ2{Im>zUkZY|lW&h#Bx5W3~xqYgfU$|*63v^7uv@~Eb8V;vf~+LyWl z$=!zO+E&au+M-k%eZ+Ls2{JISu{i4?dfYzIWorVsq@Nz0ktHQ_h$`mkE_d(ZET?C86IG$#^c_;*ipBAQ1TF;YUi zpw-3zQNB6@Da*KtyLu(sw|(}sku)sg(5cbHN}3rUT3AvdlmJKfAqtzRfFvGxajv9M zR#ZIksC;A+C?n=E36!DH{q89HBv4KQj6KYyANBT#Ci;sGLM(u!qd#WS`q zTl%Fc9aZs|LM1OGOX`wc-tg`BV1JC9*J~~zjR*(P5jBAp1tiIslcUYGhc7gi0^K z7oUaal}E-k0XimfrZ!d?rh_X7018$q94E$OMQ@lS+maty{!qeAe-4V-npA_XF$b;k`m;Tm-zO4pKt?^zSGUIj1+ z>#Uune-H($VzO#NxCZ7yZz<0}K+|bIp4|RB1C-GY0cxp0P_xS~9-tT5C{nN&vUlAJ zA+D-w&zw#HSO&iaH-|eNK*s*L$wUwaSO@ z^ql-tI>PQk`y|$H*)5CNBsJGez5D5>nzCg*%!TG4Z8`pQN6X{KZ{&_H)|jk=g(8s+>6bZf6NWiWYcI}aGg?+ z%Nda_14}+5n;A)GlBGxIa?;WV?`FA^{=Y@4VzkcV|?SgvUSe?#sIUkuKt+*2@ydhSNLC!no3&1lE-YN;Ch>s>)}o)|FSGer0GV+7m99v8g}3k4AKx>7o1gB${( z*Iv3US!IC1aGuhGhZ=-b>|I)Gf!iX662R|RQI$JLcowPRL5t4YUSDslr#Hq6x}UuW ztgg+vGT;dIm;-GRvK8rK;EdTOt~-F-X(hX0n=PQV8s3e8DrrigwFU$Z5afggn~XZ! zB;M2<<1_Ro9g-Yv~sk_6mfWSd8;CLY5 zXb|Af5&-yM@ZYYc@D_O;{&LB>k_F90U4T^}5&5?ER=2TGud(>`Xnmt29zM-!LD#G) zbs*<0fqoj&54@Mhu=?ptey(TogAvAhr;-;BIF*JrgVUkuflDHCURAn`0c}}=FPaA7 zP$0!vNzD2V0Z0U`Gw-18dAz@)*5Jc#p83izehzCdLvFB9Mgcs|>ESLJ@ zzImWyviIw=qYMGMy9h%7YBM$xk5NGSh&&orr`{$a_u)~1g9_dz%aT#3etk!=xJPFV z2aTaFgV$st)z4%iolK;^jEQvbvOW^gvts0aKa?+Ib#?JJdk+q?!s;-Dn9cisFJb}1 zE>gD5#Exlp+yo@l(s}2BRIBYO^Ts!~W`+82FEj+I4X^6#l2zmn-NWK&T^%9WM1Y9O z7C<{86I9AN*QXJ~GaM~8K^tZl13?>Yc!pkH7ZqAPb_@qX*1l8fL5wN*d+T)rO}7W- zXJORY@4Hst3U2B&-%4K%0OEGrfb#(1cQn~)M#lvo(A+j0=XYd(y92zxnd65XmZL9)}LS1tdYt+rs z&64An-by@C8a+<36kEE6qZd8b{)Z%GSWlcufRxJh@eWi2+v*lfeHIgw3Go!|@)ZbZ zh=R&ppa>%^MO~yCkG-ij_^vnL_f{>z1@HoIqm8kD$ttSu!}G=SXAfTfQ6PW7uX<*3 zBTkyT@!%1ajHcffnKz2P2ec0d6h1_tJ-*Vxs_?73l2fHMggFbVN?@&0|SQA*#~{Z?B&A zoBM4UJ#{GWr#Qwovj}IaKr5n}O9;yK$kh4s5C!zuFa`b!I*g0#;Ww z%>rw{?vt#R{Pv;uKMY;ja7E`Ep9B>%u-ZvT?peFwLhV5nujf&Cs|6v_c zc;B_ljo|xd^V5F}otlt{guO-ycW-n$fK}{)>FMcA0RyxI*?vv#isI z)EwVZJhV$FRKr_cv-jdsZB3oA8My#*(f$VRW{IWGry&oVrrF8x?Gyj5}ijHA6W_=Nu#Kbv&DoK4e4o5 zf+yH9@iqMT@a zb$w>V%FvzWQpb47s&3E^)NM{rElS?p*ai9P&baD4`9-8t4qcB?Uz@D8%iv{QiIUy2 zlANSMedBDB-SR}q6#s{nLKiAnV~D!GX2uXum5l0YHqXcjeC8f!p0gB7vz!`?ce#@l zbJor9*Fn$=8uU(kA;Y@{1OwmPt{8^>_l|;QrCyqO-m?&Ksc#~hvij)+A(Ckh7Wt<- zf0UGL4R;K0&w)2AqL#>z7*&a_$i#YgxmC*9D^bX*1PU-hZrY0UST-#Y2~A;21-5ml z92F8dTASEgx@6fUczr|}M_SAb?i7(6%7Q~jb4oJRw)=osX(XvOqh+SvqSN&yv}7ilAds^cOTAZ&XmD{YM$T;f=? zS5K6{M-y#ks6=aN^77=iLW~-KwqWv-B%;JZ{s^LLPH`%7%AGh2QT9SLex<@ODMRJL zyQZ|>hgeB8;}ZPj-L>#)fp*0=^b6R(fn_{<|Jq+fFrs>Qw@gESv#c(0B2h!vXTX&D z*1T$l(%8(D^Jplt4#CVKYPuQORyBdG{(5*uH?3rktJab1BNs|NTJl%iD){y&R2{$Y z?3k8_6r&?12BiTcgOjg!Yf!8{FJ|PusLqQR^dUsB{>fkerokMz&yI2(}qP|L_@KmpWqc9}E3y9;(1(4=V@~ z6*GEZVjvOtV^&ATxJIw@K1apDAP>svR^6=s*BH(IlWZw>momZUj7VOrGuHV|+rI5C zRO=lbJ!Au>1z{LysYcc3|4J1Ul4HGA54PI-ahFzt9jWfdNy#tqxeQK7~6+ik@{ zKwaIYjkWWM-s@7-aNkUc;89EAY}N!Q*|WrTlQeG(m@82(d75Iy8yJ8ub7(254e4#X zx-lKyX3>8G1EAfFVwod|N9ij`#W9$B76cjs*)J99moQYf0j6+maI=EiAhjT8q^Pq@ z@qgwurh|ro_9nRM+EBYzgrVwg{Gt=0<(YUjn1H8--d)1;;zFcX9TnNV?^gDmu5Sz( zeJmv>=7(itZ##A7(FIt01fvNvLRY4emsH(f-mtqU8g={*U+$$Sq-EM^4P3$9!4 z;nqt=>Vn-ewe2aFy6UirnYdOb_q#&Uipv!^Q2*y^@m^HMAL1+|11Sors17d`Dtk};?TO$+jl7q9XCUcobnar|QblPDNTvRYQY zToQAPp!h4e6rQ7`04ofHzcO^SLkQevF|-0wNnSjC@$~sirGcPNh)RuHw!hlX7vyJW zi)tp%$QKb6ouac{Gxg4_VbrwPL6>fNU`WAPvAd(YAVDy4%bMRZvf=9uEA41o z@Q1@a<|e-uJ9ASoUMb2!`2w7bPEVd)G)SCG5q)A-`W z!9I<=xxtZQY4g72L8k*$$`BOi{8B?We%P>n8SOqb(q$lhfds_zC!k#1p;YZWXOMYq zxIGE5d+aqCbcm%E8!>9u*`~P`x|AkJNk!U53?D$1JL}V12llU; zW>S&H!rRYg4e?A$X$6Wlca4g^>89?1n{ z)~jYTA-Yi91bpX8pKXrcxqG9UD_ihvW&*CX=Y%YETo?rplYKArl6#J+;x%YUY!qQ; zuBq&9(=&ZR;Kh*Z7RLEQpFTF`td86SQuJ3IChBG815EqH%vuHCNcH=FjNEP-7<4HT)Q;2vlM8a*BYKRA5kG3A~MI)d{Ue!QAW~5+uz!lZv zc<*fo1HpwxAgH{bOydd~S@fJfaP#8dT;=4=6dXq+6}q_EM&E89)Vh&fJN^<%{3ONz6hq?xOK>Vmsp*!^8%uKMS(}m zcg*Z*SUA+mUO@=Q*h^r4<+}pVg_{mw_=xwo4<2jZ;jxZ8o*YHpMP2FMW`t$} z43UZ=!-f<}7^^8IgO{zHf{~Iu9Lo3k|Ugm;&es*{H| z(O4ZdD2%l_MLJUgrPZNNV>oKQ?2br@lSeb`^(UP|XZ7k4MmwW9uZAo)hu-Hs`&67U zeP3SrZd{hLF}UCW$%KL#4K8`BO15@%#tyQ$ln-zr8JW#B_&=!RumQnIvm1~zJi z3%LE6Z4E$>NKF~ZQD-BM5LU^3$vaG&uYkUpnViKkNXHDr1P+>^9I z4H9N&l&vGJZ5@FXM|8;ZW7-2OBfj5m6;k}k7f+u*eZHsQI~%nX{|or9waT0SpRWe& zB%a6yJ*Uo1i8S1MeF`tYj{e1IYu6`VJbm%>`H7nIi&Hgw%QF9B{`{19z(G4L>X|Xw zvCW?sCUIGmsg5Dno9Jv-%?t~9D88N>ehb&ny0Lcx$PKs@ctxJ%qGV6phrVgYX*+kB zzJo}(?zpA7F$WI*t@5&?`95hMcD}F5hd0&xgK9BLp9*|lWK8T8}Og)MC2wkD`q{~w=sZAk=14ethS(A zX8;<2x+uyfyQ`;;%=P*wzCB=V)Mtk*fT{fyG5a_ARSXo;9(g=Zn{JlI}^m8Hj`=)Fh!0pgqdV``_Qey#NVHBnV3KjHe-fXeSm40vEu&7x&v| zt`;xJvu!50U^HfJX(W9j7CRNy^*m1#dIhnF<`M@diAp;tt$B!udjg|CpwF+b&h)tf zur6C_)Q1Vu8gFNTE(2EkaznIL{n_oX>x#-etl%dVOYQtLrHo4?9w z)Ly2b)HUcyo0OP~DV_3!s}-T()MDMD3abhRB6w2=9biV#uysrUv@uZCT8jeT`@C=lR3d{*T;9B% z7$9|v<_*?V&5XfMjY7G3`rWtzEk#i`Msen#+|oiVM6S;Uur?SDA~5O!A&6sZufzq+ z-5H%VvIz4%5}YbjdOupax5of{+3;DAov_r)5Z(d#e53u!kk6~FjrBAGrsGuH{PlvL z6H_?0547(FOmzbbc|1&FcR*^g9Vj&!*v`K^+u5k$8!(?=J?aVWyzC0{o)9KNKYMOZEbhJe9QDPc0Dx0x@ipTKaNNc0$V{N~Ul0l&i zn8%<{c8%^=$Jqyka!@D-g>qm!gF^ZEc}|Ez$#S)&N*PsZ8KO&U=$3x1$`tOr?K3AH6;QTJ{LkxXGysaqoXrBRS1+C~bY z^XF|ieAYK6%mn>MC0Yc41@DiCdFR_s%uyLBvGJ3ycDanFJ&JGUMFu|5oF#(BAeT0< zV=N+$WO;LWXnj?fQpw2S&E)*622=q*s}zNLH+z@IEWhC?`;KR^XenQ@1|(j^NiD=Z z5eXLcE7VBLMnd;@Z2n^^imWw;5uNEabsfSVaweCijyOPAP0qg}Dg6LcfE2gW?Y{%4 zNpmI@tVQcKIPW2)bb8AbyEeBJpb#i^MwrY4QO8A zhO5;%iY`2@2f@ljn(KhJE=7F8s&QJVtc|Ma|WTi0FI#9AflDMC}L7# z?FO2y9N}pW`%*N9dUxVm+{=rEO2wnE5)s{MY6NDoHoaM!X+}=Jn~-xZZ#B7nKoTZZ zcMYzLlG|d+5(Zggpfsd3T40v7DaXY=EQR#vs9W0fH6}{j7sqBN{g&QKhSQzu>FS?Z zo5?T4>N}I|*-lO5#^-G*0Jwtz7!I6kYk>)Xc?C+ZL_zK;=F9-68Iv2nd`T`cSztje zwf31#lOUvomWe~t%1~ymU=VnX@IB-Lb4)vYzfaf)_2S79dGcW<_5WpRCZCMSWx7mw z#Ff@mvKh;B+$~dPUhwwGXRdnkAe+MbeY7lV5PD@0V=OQobj5@2Ug}1jy zNbeFUcYyfdkoe5eLB?pPiqH$6_n>^mZd2zEHI>9$uR^zm4SG2v0;{?nnF6#}?1Rhl zA#az{mHoQrz|JOwnx^KbF}A>=1C=a^6bf&Ossx%HkiE!_6>`Xl@}&g zX(sZB8P5+s;-!vcN||)mx}D@BGI$c~2f^Mi&%^0t-+&8j#(ydpxjeTFe#vslrP3@T z=JM7{C@XX|rL-hIJWc7(A|n@HP0X;Kej=jnhzDyaQr#PQ zD>CMf*STgiCi>(a;Xl92xnf>!6eNQ57WaGV)+S_Mu#<7!$`GrjA271WFDw!5HOYdYsH6tZyktn|O6#XG~%j*pWC5!Fu(HBqP zQskUrouc@qg{bM=fv{a0J9~q`hlabBIEZ?aclUURX3DI5%{YlL8(K2;bZoUl#7zM* z?97dNt30K~vta`zhx)f;u&q8F1JCT#dloF;P?+qCtxD03`b)Pgwj9^{z&F>IJIQKa zoZ6>GA^$PnSr#B;C$htz$5D+0GaV#m1cA&%H` zF$;qp(2u^F)GelGY`Gw>CZ&e$coBVe1Q@nN`|5!t37;_!fOinws>rwx{C8tE2lfrU zy4Q9s$Q$Rcltdy)uuzG2_lg-3qU^Ha>Sl_!$?@%<;PPg!X~(z#$JZC9=Wi~?Y5d!T zxF0DoDkK|m<&%s7*o1+hjUkABOOJbxG?<&zLX+^F4VuJQJrf$0TF0gk%71&wsR^8< z#{5?3XbG4AWMuHLCrds6#* z*X?m>g5_`7iU2g|OyqiR=JmbAR0KpgE+osx`02+&+%7=}gS}=grH|T4d(CD|lr?`Y z{lR{QgBn;pt=6gojig3|*H2H|(6qOZl(5GfK>r{i^gSg_XBN|TIZXPjhHrZPwBCTi z6;Zj>^>P{uh*HX7#r;ys+0HahdG=_yLAQ=+RhF&wN1ny)R9`b`!n1V+()`UhVM-sf zuX?uC9z{%kx>3$=AYu<@l(U|=UZ%OO!D442 zF5jKkjeG>{?@`|SgXHE8jLGU z5%Uv6T$f@ANZ_p`KujB7cvZ;RH9WkZlh%2u&azVdmnl~z!|}TVnL=P>MEuv9O=Q67 zK&U&=XXC0mY==!MQWiI+#8*B!IXOY>(=WdGf{2_@60?Y>>koQ6#vcCt*|RbEpVK$5 zA^dTvBx0xISM>w6jy(vMy8YMW-P;O9uKp(68cTB40*C#>{NsxkV{*=Oz%t+l=*dQ8 z+K}N)C_({I6VxRC-FAV{3JMT6F-?Ha7s_Dm=-r#}NMQ;hvQE)~M@4MZAC93^17X5P zB;mJ=%;eX(D3&GCsR@wv_GywKaS+vBluUiZIHXSdWbG`$(`y=`0hYO~RmS$=_Y)J# z0ihq5S5gM=7Z5k}hEXG-&1eMdALc??-6vaIuxvxF*shu)=>uH0;{&aobh z#|OwlL{6llxjFOt?>-$Mz+k^t<$9^N7>agVT#eRpNvNUJrz`1nTL(ddYJ0FZ_gaW4 z?GVci13O5KRDQRmRL0gIn-ylQCCnAZuD}iGcCR8C+XDz@Y<*mAh86O;U5`gVPW?eo8Ts)xL3N8zoau>@t(+g#E z`zjVTg1H8`hC6TCA+F&rlgHyVckJEN5wy}Fm4s?EvoF}BhwZCuf~pr@>@LL87qF4~ zn`Aslf$zz;m!sSU1o~qGa9j{u7!au7c6DD2Or)BL)x6T1>{D z;c_wQZS-$?8H2CND|;6XH7UIf4Yq&DGRZ+NS*y6Cm!XS3>jy?aPG7=IogV$u+iLvZ zcBgo1T~WXBO}2F3ArVayxn2*$vVFgfuDgUQ?9}cwt~a5j2!w8+Rxw5daMs_6B1tMR z`7|RUvmG1}nk4o+RlNZJXme%`fN$Fk&X&*K*|L2UwY?c453r?Ax=LSF%?ctbu(l7! z>yt-#s#MOWh4OFNKJbC=7&_OoM#}XOfnKAp?!d-IhQN_?%mOUb<-y%#jFnOV`T+jX zQ?~^METRHx(E&fM<6@#|p?hV#TQ?v<0>-(mS{sY|?4IWF=~Zj~SY0Kno_Cc}`)8v6 zPN$+!HiyCE@Y?i_eM&D_lYz}to?rK5dACHPm&<3qBeFw^ZQ)cyOPROV!KpSl)jkiW z+9f*Gv?NWqgrYh%eq+l}`K@$){TAmHr$hC2J?+cU#Je;ZVGeDziffg@m%@OR@5uuK zeRW)58s>aZrH~+gJ~{^-59CsGnDwxMvO}{>pk$Os2+A*y@D{K4D`=KW8ZmD*uBw%S zHNdCDF>9VpzahTY>4~N(JzA7M34qj(=r0O{jA-Gj?la^p2SU8VgM0b`!>)Si$fT7 ze_Yk;)kn>E5<|{vUUMHF9E`^Y82C|L7z#BT{fV$F5^<=PT8}Z83e>8~6y-WkQ#;;X zZb=;9UQwBBouXG;Dn*qlR8~AP6E2G+2J1w&2T2m1awr}U1LI**NpgsP97oF{glCwi zQYKk!8hWTIm(ywCilacpxS8-DoxFQW=>;qZqs{Nr~|zkK?O{g2;2eR_EK{omev zeRFm35B~I*@3SJk#sB){@OyUgkHAizKK-&8$YH(JG(MiGk;q3#`j=3vE}#=jpl!}e zC8SCwR%+=Fu+{sh*qy~&>mzt@4LdY`<>N)Cban(TvvVGp!f2naCoU+KE=5lB6^V^$ zY}KRwg;M}y=c0cwzL@8`WHF{J=0)10kOn;OQsf?r)SBYAHl_j}^E+`(QF9vI;w`Bp zCjzTDdA3iU7(aN*XqPPzP0LJc^pho+OKT2FifYQlpr9|6v5!Vv_>@aPliV|?oH05MwBoPr!0?Or)x)uYQ=fa*<>j>0a zNPiKvKQN0X(r@sl%VokND2NX?RaoY7+uT4+-w!-3Qc_60$wg+K@%Cy?OgX@NTgxAc zqwowmtl!_ebb?7q$;uQ28Kw4mDGRu4m>Hn{>+Mm@z`b9po3gBkzHt~co`j!9Tfn+p zgd#J6XdmVHJW5?>OAV)6xY>xeAnEp`e7r8+oXXHnP+aSDrYk~i99PNxt)uv+G}aig z7p)>wxrGN+i^pNa1_O^PGSdHJjXl=GH)S&nSZO~sx-Yn2n{=3%B}_pE3oFa^WAsca z?)2vTAtT>NbhNg+XJ#)6eOk*7m3V#47>hAKCShgwd6CH@GJR@81lv0nGm_~GLW$mh zR-~M10z(~}T%<&1bSW1C;3Xrvq!Cvh9LJnSxB6A^*}?FRNc7Z%H@nMLPw`QUTom(m z{f(AyM+uddx{dvsTe2kaDx=-0yBJp|xiH=z7zd%`sw};X{-(2p@{}A(#>j=;()i27 zuP&oCGOx#qi4s04!&+btQ||J)n*#ir0*!-wT5hYxpWC-X4slo(TYzP;aN;^nuP&u~ zo5wR_3RHV1pNMSk*4wI1s1^qq;hDaN#|2GL5a8fjyF8m*Vz+$0P@{VWo?Vpb6|b>_ zB&At!zhk zBn@-a8^I|QUj7+HqNFLJ=(RvAhUxEj)&99*=`x`zJeK3hcaGKE@xJOaO|y7J6TR)O zzCG)T6ev$>EfnjAZD zq2Faf{jS{lHi!^34ngRh;1<_Pvw#c?_8lZJt2|wiluJc#*_ceU!r_K4;7Vo$YV@<1 z6q(}w=p+4R`F$XRtE61ch%6#46qh+0S#_vu^EWY6Q#5EPOccBR(t=`S*D~&$Jgg1P z%#mD=<%YS7EUHu?SlEt&Hg7=#N>(On2(1OiuamKeNlEk75n!5eWn-c-qSilFy#guC z5Irnsk;tUViwIVVl4&k(SyrYdXlZccQv!*N3c!`I5v&lpAxN~K*_>6W+$oJ=kNmB? z0d!e2>}t@N*08yPWtnP)r1W_Gkd-w}kGcWIEeUzjDtJf_KSjzEjj5u&BD3Gn5;@C6 zya2(RNSLsk&GIH7mfVGC27a-yG=f@6YE0gWKu;?MWP_>OBR#30ByWt zrd>Y4V9Tu5?s`%5rb_&eswi8aMU$@C?Yz_-vP|)u*}4XLgBwB@8E))4$-52HnrF|M zl2vdf$}`OvjrL+JHz4d_89IBV##qsOF~CKX!pJ2peY+;#$3=&DaBoB$=FeM$ZmV9d zk}oj1M}m0^9|&?gq|l36$G9a6dbho$xzb)oCDIHGX#}4eJqvM(1Vj|t*3;7#lCNr; z=24Six6oSUpYmP9>P_5685tA+Ph}h9ngzWizpE)YMj0+|^q7!yXs`HVvmmy1RQ>S~ zhc#LIAZTXIw^Wlj6y{8{n!nb>a|E6@DU*6b=ORg14&*Yo|FCUPt6~C(E9m+}8`8?Ki_5ht;?cAQ}2N%~nXvb7#|_!6&HKxt^B3@?>Exfe!L;q-LPX2w$E{;A%7P_LEjt@V$>x7PW%3}g-J*l6}ny-1VJR>i}* zcB-%t2DLFNw-Qd{OadZ0`o?rXBrU{)g+0B-jZUvF;bnU-oWp-uX&>w)wr$EZJE$rq z0ioS*^bXO(RzJgk1KbYdcq<{=bEdx(+$GF(3^0K-TbUp|wcvR?TGCvt&_;7qjy||! zH*GTTrQR_iT~I=9tF1SX&U(y@o3)HhfYl40XSK=?HOr73P1|a-bfGt_#pRLCGCXDj zs&{Ztbz}Fz&J=oguz|rHy9H2_IB3EdD|KD4Kj^O=5#Da0 zbsHs_$VMhvzjWRhz1v0*qfLQkVVa#$+F-P9_or8vhDjf0B15)^&xR>(Xi00l++a7Q z_F?nMLnW+JmdUC|Jd3qF0I>*$3|T}G3p`53Z0I5(S0ccWC#5P{FiE5c+U|t#Oe&fr z6;`*|00J~l<4bZC(J&V7WZxSL=rWhYJ#ssSVZHLMz{Agk0lg1po$}6+YxGQu7Z^v9E2UwYMFVq zu4lTFKCFHVJErS8>eEv9m8`@7tB4TRX`w_)6_3CI$Y+jsEiF+hY%E3C5d~EQlqIuR z=1ewgtAYD|osiD%!liL&5VlGsov;+5(Z|eyZ!m-eJn1R^=?TrNNh0ngQTL*>dZG3gwp=sAjpWhVb-?ti)l0pUQ5TMQ*&}p! zq|f?2OA`J6K91dC`u5D8&oj*K=hB)cO>)g)oQvf$R8i7<{ptGbAq*tHV-kXaY|@Vp zp22N%ma&{FaD18yo_t2=#!zcVVQWIT+W?)+c1e43YG>I^4`5EzJHQ^^z~zF%P*2G> z1_Hj6FJ)eKi+vzW2|^nJ+6T~cV@w2Na(J377tWUh1xv}B%GA|5Rorn0^>~&GDc{;| z+sq(#3ZVuDJI&!HKHS8crZ?Qgo7%78ChqXCt*2UQXLaMEGwH1lFnKGK2~+^hnv#At zP_l7Lqrf}w>1v!YW$IB`pTM>K?$WoA8M$VWxMMkPfQt`Hnn^CQSCl8A9wEC`DyI-Y)z#w@ zdU_#ChHjQSH3TrVzjU7_Fm#!-J1&YaI+!$uLZOBsmAR`g_`F1RwV}!odXvZDw_yw^ zrmB$jS}PF*H0;sEU9SB@Xq=dg5h||wmyoNy9iACWw#<$t+hxd-$C{i`Z+l2aB^$iw zHDg^fSc~( zgVv3q+SOL~>-A3R#R-8F!CS5*qpPuEuxL$(4Rx#OcKY@_{2tXKjFD_u3eCDVO(M8Q zg5)dv08UYcjh5DWa?4iX#zA5NuMJeN?G`f*YVc+~y6i6LTW@_gC$=E?Dz~*kS>f6M z7EKsXklV1tx>*v)AlGCQ;gcuZ*uAjXHQRv<<+9h3fN$BVqoVU>!>U#ylMBAI@V#)( z&2Avy(uDWfMeYpovU`~wk+(wW|GR(;lAilgw{yYdTcP0nkFrQO{T-Ib`1d&Y)iBEo zDsA)}M>Uo*ma<%YF3Dv^L~f|N#St)r#Nk=@Lgfwb*<-`_%z}%&wt{;bYUu;-NcGul z*bmrGA%dh#3jr`cs@Oar(5l{2md~L&ToeRQCEE z6Rsimy`=V^IVeJ}A<^mM+-$d4(@o6{2aH-rN|&1J{9mnHzybPyJ?^nQmt%5DB+upv zbNjGnJ?C@5-RcqPd(uw@zoQAmd`6m;m~UdXp?`*R3j}%9DnXgs_vDtXo*b?7rQjZ) zTxL(I`kyr(RO)aL)9?g-3oKtuOs3Yl3fxb%eaL&$t|6$#PJ5P6bf@&U^{@BP0F)I( zr-{=~Ljj$xFUJPL!Qe+3>w|3=Z>;@;WK)Bn2nQW=G zPMt*Nz+-9FY!igp#owxuje`0Vxk;Gb2u}w5uL`B58;`GQRhUN3tU{$C(ISO0F0lTdAmGM~H_wODt?kunHE`SBH>=8T*zXqK_W zDa4GfvDPtDrD(K3SwM;ATM1JKk}QN&B&E@UXJv`ZtSqP9kw38%t*T0h*}5k*?Sa9j zHQTcScg160LDs(MD)RzxYU}S;sC$MD0BP_#3&IJW^>5mCuv~JPmOe|KIdYkq7rg!Q znszfbV3`YXPhvst83ZC}JryzwWC&u8GkZwNi$uE2MapfIyxkK!7vN0_FCY`P#C8z% zpQjpd(+E#bVzy*iM?Ei*kyq6|MS|>1dxPWk>7kkrETULVVmLS4g-Bva*awwU@5dvN zrdp9L`Qgi0|;^7k)gciVX&{%sZjfK*bUPrv?I)Z9hVf21-67nLlu`KDt^queHxLPQZWkQ>@`mGf2nB4vruZ z8UHOCP|0yN{=u4=dSLk*{(9Fc;@Uv^o=dZ|2}#*1y-g%1t?K7)e2 zVX#5w*)@xX`J+*?97{#JHKWCXVa}|PrnZ!w{eUs9d$?PE$891f$%`y&U$h|Aj4CRb z>N3aG^|R_7%{|;RVtxYu6N?G$U>T=$iGCk+= zHVlSqT+nXNO z?+X@rqJoC;?zfdF$h|2rZtf{JeEE`GWU|Ov1q!tGnXWw?cxxdSSZiLswD%Kl$30D$ z>t;%C-}~MT-j4;@OV@3pUAK&fv6n7suHd>AzB$WREGu;WknOrVFK0HPr|t*mvV6#Y z$u4WiX({CHz*i+M3ptM2-LZ`6vm+QnNVaZ$o>Y=2&z_9Qgr_{Ad4dJO%BpDgIloC< zg3fX_Q3)ip0&ombAP zj%&%+XBT7gjk(Ggv5Z_z&tLCpc;aIcHt6W?U?B*(&--P-+FgIs=kO^A=C>zw-%M~v zZLL#rbmK}Zfzrc@D%%p@uj>hl|+^><9pFRrdH&Q5PG&R>$R z%@??wZxpG7~iO~5}{T)S8r=d6#rJaU| z9KrGo)ckmSFov9rgYkh?#1b1U>Fw0WrfM)Qep7wpjR|>YlUw~78{df6EW<1-2+M{q zbnzCjTNP7xuqJGC&hJbu$DM7_)o_BB*g2f`8hHJB_i^B9>@f-h@nyyki8MttnnhXi zEUds~$vJkLLpNvbT>5VAiYP#{)+4*dU+v>-H`MaaCMDe;x&YT2;8(qnqYcECwDq8j zNINbEb$oqsdj95OoW{GbD3--b!83n}hBX{*`-DzOxam^WiE z_aMY?F&KG7N1DRBUu)E&&y7YcEJ3Q7mD;>o1ZNX`@`>+RDO{&Umw#i*@L&NxpjtR6 zTuYXx9CLqa+M@rC)T&879D#RLBFiJzu4cBQ@OQF7&5^D=o4dZY_USqkT;^<4qR-5D z!k~J$m9_1@+pIG6Sj(KXDw#Iq#`8I49@ zQab#n)|unvDfy0>??ZZn%6X*7{XsW7+h6@{aUcyH$IW8H(+KT$l^`r%ssA+R2L zk9X3qx8Ts$+Fojf&W;?1KqXl|Pl;Tm3D0gT5TlteMG_g&KiK`7GwRPTSeFhv?SP%R zzv^XW27|IbXMeS9!UTJ`m-#18V0IlE`zM}xz}0>Sp37w2SE@zt^R(YXt z2LhU3X8KmrRk*nx|9nI;c5lYS?fc|Ej9C%TK9aw@42CbSpb9PGtFaYDBFk9R1)3YS zBtM0-J^oX+W+}y~{&ztWB;VV8nEC4D4& zk^|%9JRpZET6+C^d(VMMip*74aGW=|9Iwk97~hcg>;!~e9d6Mw&zI}tj{gdNBd@uH z(3~zjdU%*i0}Bh9+m*g_^5u^-j`No&gJ(vQ$~zOrEffRIN-y@IS}hIM(aqTv^diZh z{&)hK+#jC*@x;6MA2;u$wjE{HL~|jHGy9klZ4LSPJVoS`qjm2pc_WFcr21?VYKpX;vU0KZQJ?k zT(&IKX_u|J>oW58)uiUB+wjP>TTs?Lb-P}jvRNwy9?0J5)Ss?hQZ-Qk94;R}q;1YK zVe*OGQF`V`jV{koe@9feVzF~@T@XdP6_>nE2Ic`iZ*>-ZevtX(n*D3W&!h~gy|%vV zKIzWd$>2LO^VpWnFPlmcg>ksXS4{~aTSckjf1YDyGT3ZEqo&$OHjaj-4w_c`{@!1` zq4hFTP<~}6<_SIYiw-ScSx-K0Ay?OhWRsMHN-K3u_?nxg{1BXZXIhg7BYL!{tBKr>n_+i22V-}R_2;~t4@`e94+%X6b)+DtTQHW>-nWn^=wJ-9@VFVD9=m?BsA zBJWeAqyEZT1daJ^J&CmaHuAX6-UQCx8s1*p8T4z!<)yXf5%l{(O26CM5qw-nZ+j`( zQI(+-o8H-MRh41q`v-3M9z4gsUbpzZPGi!Z() z01y_lh^OnF5Pa(2pFJCs|2ci*3+X2YE>~j3DjX3kh}*F|BL6jc_qM8&YJT;0Y%0;~ zJyMc$OwOI~JJ;4n)uIz&3CbnUW=?5aE-h!U1xNUNVFAeX9!q#fC3!HXpk3J3yumO_ z_)aNFUXpad4jiPIqZ9u}mp$@HOj6U^}(D@wDMeagYDbtK1;XhY3 z3>S=CUA%!9a8QZ0DyCS5$&Fz(!Sx2L)Zj587g8vw=%;%&3na_J;1EnQnkFLhmlMDi ziPT_qEBnJXt`wZn27JYFCPxE|q@tDNR5R9@`bX=a7^1c4NWQ6X9*WMBE!ebO7y4))ok2fpMQf5KNm z#oIhwvJa>o0Op{;Uf=u#{UcNo1~NCWM{bW^8K=h7Sl{4OL6c-PVjnbn=65VaSlEUs ztp*Q)sUzIf%bY2!+w%nHkqOAicmj_Gb`8J_8dw&Qio!{Ha)mU-6v~el1Kl7voSq)!R2a2<6W7j`D;~ z6MN5xP$#tULGrN;u4?*N=(uHBp&=8%YoI=b`@w3>Lm5puxJLWqTVsB#?~DH9Rl=BR zerCC7GWzt{YF)n9s&$1ESFN_suHoi!XWT~s#pG1&%7qAq-4ha zVvTdm*W>*d&u&bi6s&SGGlljIAZ5eA1ADjWo<&xHl-BIL`(LkcrPLdsyZ-ec8Hf(@ zH+0$4rBuG>;y4gG6@{|dvG{ucxu62N64v{+XXA8_t4TeT6+#c{u`fr2(P*Jnun{OZ zfqP0ToK~|u*OM)MW$SXM2i1tPS)G6h<98`ztfUDt3hfbe%*=@Zwl+9(*6g2Q{-Ju@ zCPH<>r5^4rTglB?gvt{;*D7^f9}g4iQDn|H`p^?_wIGFGGE-ccTDzW;q`u%$D97Rm z%xikIJE2_cqtK!bRg+8=OB^ng7|^q2QX(KTNojt|;!?8;U8O5cywW_8TcHP0cyv`O zuo-U1Js6Jlg}@Rkpg62rggfN0(EuDRw{2s8?rD2& zvpbu4n$y(BlC#xCJq^SLZXEUj#HiFjhsvmxUIw9-_$f?)Gl&5Fj9UHxsxnBbSD!%v zRlRyFgReRQthTTuw*+H30av{`b%L`lXTVGvi{SR^(~YSs6*q~_cqdA$pI@ws=<0~t z>ea0;!fP#xt5?6Ch_4VZaN9ydz&5586cV=H!tY_&>d~Q2-O;>uGwbCMc9sNgqf3W* zj>#J%@Oo|@s&!x~l0uawP`8$x*kEGTfQXf>ZI_ia?LBQH62pbCd9}ZQ=xcHBV{H$- z4q@BUk{D`T-_i`&Swtdmb|1Oa1h&2fCjz?-DNBLv3S4R=A6FewX_@U6f>!<|AVe;63e1766W|Bhlu>#fe+udVbm zR#mjdgpV4Vu;pdqm*9*eG|Pm-V3;E`#rJX~k}{0g&CAZ6?6Y3FI@m3-JKXvOql zsU)KXOKERCOSe~gWv+~RSw%7v#qv}VG3y`v;byEoxqB8EZCSOg10-g4Dm#(($|)d? zHcPl-Cw(tJ#Hj{QTF^AJzr(frNMzYA`lBReZ$-IhILGe3Xl%P#WEw0n*1FPbm3)?% zBdhlGy#aUO*2*P-GI6l>&-ZGN~4U(YXDmZ2jlTU zpVcX`3(U2xP_xmW2+JZ7+Z>=7&wCD1E9sRDz_n-rn=EZT{Z9dJ1PFX7Q)lu1@{;{WE zt43pbgJ>fX6?x7wm8?FpW?s@SJcOib^#v`F4behY>R`R~$61L)D~~F3jQ%Uj?#La@ zz4e=#+SbZ-%rp{QML@f|DAcl0MppG+O2g_b z{ziE^kIMT}?=JQI>RESQW4$e@QY_D2lK(yY;rG9co_=|F`2ES~&;R)S;SXc@$M2qg z`Sch2AHRS4^ziWezrFeT=IY`f{OK>>XGMC8|Mkn^_w3>yft@^k`sM%WIndg@fM;qX z@)0iSONfB%)tx2xZs)oZ{S`b*zY79^OM+(iAL-z3cm3-`9`C%OIgcegwi7fO%L zvFuAv8#BJtVqP{s##Du*JU%+h@~4PZ!xHX+y$*qDpK+iXu5`^b0P$yi#u^Z%hVf^C zorXrKfr~qM>km{4uh*H}_lp0>z8e-J~ zCui`vsz9}Wr8qUIH(10@qev8)x)HZ5AOK~}max$SKuoRM7D8+ZV13x^?qNgNXc2RL zjo)IA*;id~$1G-X`9t6qGu%Y!*^bF6$yvmg+>}Y6_;=iOrtonnzvW7@WQJN1&8o7= zxpOHtm%|-~jOq|@lzT22kqc2IG0E9KjXye+;p6;ooYB;xhJeK76#5O6F1f*;c?dqm zJ}h|-n|;C}k;U5>t~UCpR3>nT1)EsnA}>?b?`Xnfu2yA47#=Tu_voz5UNyThWwqvn zE|sd;W%{Q&Orz_Er}$y(4K!f(f8lToOUf^0BlSNl7z<%Xz4U_M^*d8^A}_xid>G* z`}vfVAkW}UG%4BpwIT_lQjupTB;{G5LQ51lSC&7%=y43gFDqLXHE=H|@bB`L5?r~J zz=c`wx252?l#Zz({eGVNks)G_GQ=2 zsaQrcukW;uq6?1`ai1Yn8zXam$Fj9uNdinpU(x86Wik0iTH?g3jHU+ocA1Mit~V!( z*ZS^=ltt4t)@8P((^0M3l~RT50T`(t?&3cZP5(K+i&%&*s{7n^5_ewcrj2RX<#bMk zDd21TzE~MF(QL(kAG63LYa7KBx_@v$V|#R3AMmo%PC1)T!;owa@@ZV>Vb?Skl6(cF z%FY=m?XIkCrOP^m?Nuaq$JZC9=Wi~?X}k-_WKR&S z13!XY4cyr;>oh4=q9FGa>ns}NyBofINiH&30K^VeYoF=V(}0zP;oy-ALtMW``4Q0Y zc7>MBr~Z33not+wp2ULOL&(9MzPo`}Us`L5vzW77_(>g!beW5kODk3koZHSAoIIuT z9@`NfFO3dxHt|oUQdyXVyVq&^?Lyp-lo%C~jkq`VEoj2GYz2XU!BaLb)b4&tfRPve zu9c1Jug<@bU3Mc!tv+TdI*!-v;C}8n8fl>xEK@u}%dYGWICNPTxt!sxB4#mQ z-{d@Il9c?Z+JxRNGoE}5>V~>+LL9jLT(=+FDcjc_8{}y1#pgN=aoaf;d)*fg;KEpA zknooc21(XLko zUG0ujo(qNQ=ebBJ_XaAgW9Zgiw3BiLY{+D;%6^FHXeM%}#BZVRt>!-Y38A4kRHDxZ z8oIyARB23(UCyQ$VC0Toq>$wWp;JaO5Xmyh!3pTZvKR;-F01?iRIB`!+1FyuEfARA zK1f)~z4d4ZJ?OL7gQQF@GCEECp4R$?E?*(95;`Xg{^L02QvW}hOkRU0rO3=(z)J=2 zEg1e=W|SgR*!`>`qw59y?Yu*H7F$42r+%DyqBB^XmgzSbY&2!$X2GM|E5~^va{af= z`BShFr^kCi`^-%1s*mBodB;jDtC&p;mEbojrHFVZ5HRcouyR#mECw;Q`|Lp337;71 zQ4LbquD*GK`dNn~b$MoKv-4zdUv*{sH@J%4Ith4oQsp-7<0rDYW@fRi+PR>33o4hp zJCYnEl|3ksy8GbXCZjmE{$2{~_9uMaiSEmfHzBsk+_L6a$G;i(67ZI#SQd#d2+bk$ z&i)7d{w{7PC&woI4(=xAG-r1ORlaJ-=CZpQZczHRWIi=^61Kd3h7`JLOVILmyjIw1 zAH}MIic^`Ss?flblRv(AF(&6c2Y?}7B4}4G3z{=fMls6??xfvp!6@;bs#FcxRNa2a zwJKD*W>0OIEfty_6&7IF>0M~ZK)>{6pxfN?4tfUVo%hVeZp}9D&Lj8t>*kp6HG8~A z&bXTm?(H)qQ+zWAymPtQ^20kazy18Lo)x|yRFZA6NCF&fJu|ukGquB$I)bvY_mzv? zBkVRT?>ac!x<@5SZ;o|K-gH2>dlkmm9zY~x>m!&k+c|`;U9}?XInDt+_L0S*@N;{X z$k=wuV(yo_+$RYjECC=mJGoCWO>a)}W}b0Cj}4i~o!P|!-Ij)cUOC0Vj4mvHo$fQ5^wW9q;$Np1Pu6N{T4NkA z4^Y#@Gz1|7xNmv>POmOWlrzX2rinyQ=(Tz_hTPs>jkaiYI;s>)mmN5`gA?zwcjE2E zA+RV?nvL`c1~;A~RlO8g7b0U~&MJylu>3(vTaodf3Kmi%jaE{5kG>Mh3Q*g@sus-9T!E8 zJu5+;n4g|#S-BGa^4``mU19zMHD$w^mAfIp$Q%o6m>U7k|3-J^a@lhE53DiX$)%RH z-+EKb>PNtRqpmsLP2i_(^+r?W05dpkGRddV9sPP^mxcCMtv8mw|Aa@Y%k%y*1Nfem zKC+A#(0?aiQQ5%;-_VTC(a3N(x%%qOQ@vBbdUScd?tyl-E&RaS-P0P^NYG3AZoK&b zWY|OIrj=h^&{PeQ^J=|d1qKJ$+Bnj(kPzv4m&UidO{6<0pc?gd4A%FB5v)bS58gi8 zZ9?}nITNX7$IWnTwdHtnW@At!=JDZMRxgLk+K4U~jY%Tr54yVKXT{)nm1wNkp6me zb48Xkrzuk`M>l|P2;IYgp)eXVUy@yIEUyjvrFjbvvN5jHI7Sf!1I?LSicI>7?Sx%6 zjWc(`q9W&Nh0L&rOAquwgnHHejBH0E!&Y%B-JYqnS&?rt$&?nP(}ZTXelZ?+FVqi2 z>L#a|)S81G=@yfnKu@Ext8&(6x?T6!jYV9aF1bn=m5ikHR%<#PyxU3oE+j*59#+}V z8aLF=;L@%H`jfx{)Y0iQ7xxmjp&*H%v3oeJiwJM`bg7jk+8q>v>H4Udf(+L37~>A1 zOIb|uza(5hfJ~DvcZ4-<#O#hGde4tj@iR{nIu`l-m}R4HCTQsRj!lnskH>$-^L5<1 zj{DedAIy=^3NU(X`?$*OR{Q(No-EzZ$aJ)?V?|EErOM)}Tu3Hl#%HsKBYN|Uyq{xP z;SDI&a{q~6B_ENuVxlLum!*BB#l$X)3LR{&W-xQd=3h-T2*oVJupNEYM9$}3S#!Is zZEa!?zo_Rd>v9}U^isjR_pOMpRLyZ?~}>M$sVc}pC-xnYro5))km~DnF1`Z zdbrO6)gnxRfU|M@++O_0RDjZG0P^wX^<+%GTd++33pGX?=2eB0 zADQkw-CFyS$;*hbQ+8)uzsOr_aKX`nW^;eA3MIKn5`7gYmN8Y8(NX1mK4-blu+Z(A z=~$waYVnofhG=1Nt(tvGKk&3jNmis&mg|X2o)(Fs854!R8kHTlOyHFK%yMDqmNIG! z_d5zmESSs=l!*xfvnJ0Xk+yHk>a)#s>yAT7d8X&}lAL%*+8&y+q;qz|RnlU+sFCvq zQ#0JVkba#g{eJWKJ5HubWbZqXUd)@%#^--c*r}2s^jeK*zAk z`j6Y2`5?9O!|LSspmcbB{00EadazQpwXxEDz;w2$g&$rAe_wHA!KG=pt)m|T!MCOt zLR5pFg*+Z6Q4jtqSq2m~8AU`A6DybGOHQE)tr};r_h5ogYW}YN^DrO62ix zm$&m#Br>UJrabuCoe45MuG7AdUzZu+ zc#$Of-w}3TJPdANH-Vr8a&dP4*9+flX%@=`y=6!Gg=IMqBJ!B!WA~ndAF5waJUzEv zU4jU#zjb;3>8~cYY&GeSy&ZDbTK~0}+ppEXQ`|m~sNF)$1~UF`@%orW>jPvB1=1}{ zSoRl`AO^d4Lx_+JMU?6ij())yBowz1h$;PG7B7$yyfI#6$H-KbJp-?=W~x=?B)_D) zc|Cddhc1V?cblX<>p1~*7w>Tb&v%*tiYC+^&iY#f&qStjk*rl{L|p_qpo!}R(SG2Y zPAa4_!^zhSi$|o!;Sc26!#-P)(+!qp%?noLaFrcaBKn9>04&X!^auYNNemKQvrAKR z0T5nXK~0Fw-fJ zKQ^y5gnLIxA(hsbt}Qa(It?47(wwVBF&#%Dbp+J0DT%69$J0bik5j5U*5XR#d|D`x zON^2_mV7>h)MXJczF;t`I#E^XS?3~IH$YX-5g z4X3QGTnI%ICt{|2pDaZzG2I*0pAvC3<(a0{${++cr4pcd!sbZvTC{)6mWf#DZM?Zz z{jo5m}-f~0O(59jy*9r%03f8bEd_!#2U)Pxy4#!QZ!|WgDlR3kBn6j&UYMJ3w z4f2SbCM;JeBnRbKW|n3`UKa_=%buI~L6?r2P>Ip%Fo7E$m`k|%noSM@C}A^Y{geqT zYiCtx$#++&kapOu`l{VKy6ce#?6gy2rA$f>W>sy{f30U!n^$dFVX)r8Wma|?|H}=0 zAdyI_q-EQTJ7;3K-`6ZNMZbe4^={*}^<6wpnW8aObUZJgXG_H8W_x~7g(#G5MpO4n2 zz$Fm{cq&bhB!IhWuC3NKaBH_UUSOx*wY`0~)!jv^azaqJm*s`XdWZe zY0{H6hOGB8Sx~*_5P%v0zOkDzxuDU){uQx0n#N3tEYg(bm8nm^ZBBR(O|(nF3%DaR z&9e7Rp;hp$-yk;c3rM$aQP^6!F+>#ZF!{!9_%TKepJ=beP#g56O~ub^8)|ErS!Da! zIZi*p=h>P4!x9+pCh<-a>|oL!P1$ZDRGJs`&KAOv9B`eMEPBZ4`!$k{)o*+%_x)(L zZ{u$b+pGDx`U-tI4X14~;Ytql&?R0b@Q}Ssr2jbI`_Mscz*?n zhcOv-p7f8Wh&7*fYrJnh_1jLo`HXj+?vYW**Z$ae*t?Ii{}4AG-_2ohaha*+=!-;gPH!gV)y| zG<~{{n)@;NzyI$y|MoA|q6N*BJdW}h{%2Gr9p**CgS-fu>u(QhiZ|4TxboPAI%gOr03#P`{6Zh02J zBxi+GBE2@81hQDcX!9Fuz|`!7W|_c18pCQLbZjOpAI(`d);hzq;7QDK=-ht#?qvMz z4`WRIwbL0_gRVV`g!qk##~|xdYb3yLG>g(Bn&+Zez9h{LVV8E~RIxdr0O5a+NCU~? z7o<-m^sWn2#$KyMe{1;hH8=OhWm4oc*_Z>e?S;spK6vf>x$Rr>Y+fWZUw`>G1gm$evsr7hwC)2^oPYp z8l#5hT-iffIc2wW#Fh%eL4%UAaa)EB?^yx8=OGZo0v}+pfs)WR@a+0xa#LEanv_YT zyW^OtmOv8>ibooNm0YCIC(Gic;F)s5CD!jF#Wdwgg0jUV)cqclGvHhhorp@?m|SM$ zjHWC(qb=q1_qhO?(sHB;8@2>4(_MdfGn~LMzn1mFU@MzqBxa>Q*lEFo_x7$CxAwaZkidvCQyS8_}v~*=l{EIjiQ`igQ4f=1?!m z{~n(Fa(MXt$>`7j`2Ew>(=W?^-#h<~oqsQ!f2YpBrTcgJ^vkFJXIuMR!c!gwI{8{Z ziO3q2Jy$QA1dN#4-lkc5gpyE`1uRJP19%Z^F^-KXjThZ)suwX)b`<&XRlc{*H^n|+pgzQ z6Nx52+S*J%N|kHPt2KDIE^4mu^=tF?#Jp_F6`^xwx^m-2+RsK7mkuvJ&!ANJbmgv) z)l*spvJIG0@VOBe=3h2gif1;wH(iQMOSEH6W-S2>Ax4%Et-z$!;rx<6J)~2St6%PE zo<03?>q7+$B=Arn<1CMlCR+Vqv?gqIM8($^*65Y`Pg7Puu_9mlVr!R#OjteWsUGY9 zz3uZKhjtO~@9)(dJFb;jJ+>YF2oyMOiWjp@F!3Id-7bfLcO6(f6Sfc5C~uF%f?tBqT4d1Udz|@u*ply|@AqY< zlw=(lzJamq8yDQx2V;z)pvhHmhGK96T8*No)sUdjT@2o2KtVMG1}XtL2PWWMHoEYXiHLRPVwKbZPcfSQ8{1GYhV?QGQjj`n`c=46UOEZGjxHWZ3X$0y*2bJ}OkY@$6OGIai ztwo!e=s{b0knpaomIak0N*MLlDI{SU8n(}6NimG)v9R9P7*u4quE?RVGL)j<3*SV6 zwxw`|fMj!bQG$pJ$4^zP|~K0t*POdH&p=HvE3I(YJX0WHg(vdHmehdinckvRu&Zy?*Xo zN~PFAIjo^r;2?j*QP#l}w01F{=WGtfb{geEN|F|d;!6YgWU%XaX8dSwE~9Jy7!Mx9 z!DCpX{@^kE7#>515logWGE2AaM|9)-YR7hsz8D#R45RIQ6Msfa8&s7kM&_9=Rm+*5gp2{I2EtlC)hv`#YJZ`pT&1*l|z$%035bK%SQ9BzrWXEvc0Et?>_a zJ>|W9_h2^Y}EN|a!Yv<3e921&y_iieJThd?uMXa&M!BWYP-)X0a8~lj7C!eW$O(aot3? zCiiu1IUydol3(rYgU+eGWLxwzy#^C6_D6`RwA1(9FsE=2xYuS@^Nw%r2M?br2}2co zFif$n5XFKc-9ymC9;^Jv_5hG4J?IyT5%w^^C&ADs;pku?*kE1AV1bzrJD`Gv@|+%? zcQ5^#yz+tmcW=LaeDluq{X4Uh{2l{P0}T&a?Le;lv2vz;-8SS;6VqtWaxfH^XY+M^ z#>Vuu!Uu=2cfSFui5&YfLsewyZ;T3`9~s2gAii4LAii3^^+4jwZ}`IYFy=2axvh*~ z*am(x@SEM}H}IR*Z#@XV>FheDibgC`tTl1I!N^6<%eUY}0LYEjehnl*On67ud?B$w za3Ug_*j`WPEK^S3Tl=zEMyCn;>&?y8^PpLMXE&BGuM&9IV>N1>82Ik@V>OzIBoX)5 z3?m=ID)*e(SdH?q=$TYGE!Zdi@edz9v`!3=_j~=l}R3>Il~%I)wPRT zs~@@0yJ(qn*)cNDR?5UxEJpCS1Wg?NLbR^l5<`ip%~Bo?Xu3Fw7AnWVERBkV(z6tRc3K{nJF zYC$$*jCk&I)3&epHD3?ebwi%`hA!73v0;wSE!^sXB$wxVQGX-*!CqQU<1k_0nTVS% z+&$Y&opQ-pIrY`6-louG;Igl=v-*hMucuvs$!@8CV~XYe4Xp&B%O%TVTW?5- zTb7Y;u3vlkSP;RX4pn&2*TH*y;eD;>lHs!W6P9oH8Bd7uTj4ZpripJV9aYFl_$v8p zLz-vwxW385(?<{9<15#%H`e)^>(^jdVg@I;W1%%cKo0n*5m+ELfFj*_rq*QB@qkw{ zk4lPeYbJv_`{B`+amq7tW#4&tt1{SN&qS87Nc|OpJNIMXY`Eh6jjb%^pV1o{G^BP# zAWXOE)NCF|uh~MY2|V`Q8qH(YXC6SC38Y(?Ci5_QOrRDMro)8P9ro!jLBez!-KFkw zy4v6*g?Uyw-;wOr4Mnu=J9FmavM+3o;@GH zXJ6$az35qv;ECfBw!UiUY%K`CYrjobZao8XM{`c636tbd>vKO|UBCPL{~UpCH?jXg zvzWYnhyUeCcIe}leaIE_og<)_M=x6Qp5o(LCvtT~RD6NJBZ${Cedbh9HYddebdk48ilO~f3_J46eb%^9IdQkJN1jv{F7x?gm` zW3CPXbzj4R8epG8WnNa!`-$FzZ9COp#7cMP{c*DOoZu9ODb`jw!^j7d%LN!FTKQZE zsUQwf&+i(Uv0$I`d1mj`akyt_&H?JxtL# zlH#92%V4M@-9BOPX3Y^=WRF2jzValrDUG#Zj;GF^wz~%=2ZoVdqD&yu0=5izC2U|1 zux9-MlX&N=z4KT%QX`aKj?XI@Y25T4S}hVYcZ|Le3NJY@(^*=R#}ig&65ds=Tpc#7LTGu{*y zp3;GRTUV2?u(6}Fp-B~RN>KTW1>A+0Dg;?W8Q16SE2WPyQ^!#pqCO;bhCKHfIdV9^r7DL8qkOK8~4|Ns?p|g z-4abB=5wxS@-AXD3oT!L-t_F_c+F*A-+pvpA`zzo(pc9 zts(TxcZP3#6a}Q+wZsLi5V9pxgyLgT^K`{iPp6W+$GRl%hdR{56>7Lb?MT1j3f20p zp$_$9twX)9I(^>nKmHPXTMvIIPWNex)9tgcT3wN9*PGT9EKsCs>fvts)KH_%H`^gf zVsJzL%G{7cl!V)cD2cYg4p9;Vdm5r7pnYb%X{gcWw86smc-h=wVf%Gi*oGQyZQ~wl zv^6|7)M&%@+0bub~W^(}pr^t<(7I%dlPJJD@`7m-LKtHp&@|fh0VWie{0&e$HdAPS(GOPW{-? z{V$eZ{$#MXK*$S*8E?OHGgn6x@~YawheZ0HV*7*a5h5w%y`pK zac5BP+_4Y*^H;<_2ldWvgL>CC*g?G;*wdiiLHo>j(@=3|sJMe|2xjxQA(-vg70mWI zSKQgQGDaqY3u3&Tca#gw7oi4=?YKB$0sO^w?`YDjb(D~uOO;Vm+LMI*gLhZqc{lnle>lK}w93`5b z=L$Y$6Tm0e-?q-{a-NAC<2m(nBxjMx<2ISq9Y?o=bKSnP9%8u;f7ZT6U9f265^H@m z1!=UasQ(J!W4;On^_UL;oYR}foFucIgl1YZk*8zwE`uK@*Wb3^_D+w;>FMd|5qa}> ztq{DOcxms_xV6hFaOS%O;2MEC`gU^U2`za(e(`5dWQiX(t^L@H$Z7j8r>E^dzWICW zPj9ybVe?nGYLiA=ig6Y_4;AAVXU~0N{Nn8SM-=1EV7iJ|p{zXpu6a51ux+f5#7Z5@ zoZsn7=jBydfr^_T`C28aqIqklbmdm2GT}32U8}NBeZ6qXQ{lHSPv10l>XsjZk$`1l zLGLhbjz!O3o&$ZR?9$FgAMlh*ea~oSA&!2sMza0t3@m!STZODBVp9IC{lZ(<8|>41 z)u689cN21Yb?LjK4mBKGr}C&S*Fz1*mKl6jH5`cnJq)`C9K6DZZOK1ie@*~)g`w!i zZMISj_@&^=WB88ts|$(*sq)m7L>{Umx_YV**vtW>Wxtq@s+JVC=zyh{WeTf{cb|8 zMZ(DG_1hytGeR{p%&6jb488YyYdh*|M>#n!&b3KY3>sLO*0`=hOK&RUphdS%WzeDr zEqc#d^tKisD6V-X(xn!Kjm&W)O)n!enn-rkAS)nYkrY{EI%F})nOusj5p1D^S1g#zFpfchz{j(W zxx;ZrvRV<*#)?jZ0Q`Px|v>q4@XFOVK39meFEMIudE7JI9*69}-Q zQz?={F|7zKm1vHAWE~oYL?b;nsyu}^mbuW!o22xXNsis-o5?hf8MiE843AZLsJ<3ro-lHjh$43VLe28gDqJY=OAE&oLyo?F z_wMW07eAg}UA?|MJH5Gl_x8uL>x=V?w>OukubUlaO|=&4^osPz#&Z}ZSREa`mSuCE zF(jfYl*skSkKMGLY(7$A1bvK_2~{(Zr)?xVB}=eY{8Vts6$Vb>rP0jm)|`D-d~Rc2va!8+)t?K>!mYOA(VhPT}qMW4$nAyqn7RcoBml6?)vwzPrxF zYg{Oh@hxSal)!)@XK$Mk9sdXZW2CQ^e>wgShZ-Zjuk@eCT1@_>(J>#O!>RV_OijLk z2XzII$tqLy0}3rJZI4`7E7S&Dk8z)8Iq6%fsM_$_I4UZ{# z z=$uIVR;iGHVG|O_xEVuAT;!DHn+>jj;|> z$eCu{jTxUd+SBx66SsL*8USOeXwQi#wG6!?LDetM9SN!#eY{JOI~->;mBtl5pog`~ z>7M>IKn?d^wrT(+^i6fhG8?kYJTx4#%%BaRgr5kMaH_|wB~6&M;Vb(%9Pm7C6hl_r zc-<|HVyKCCG{|oB67g)l5m#+Gl5K}Ngv&3fl=mXL~@uR|ZSlL|a*bbY7*Z}J-_MfdVpE}v@pv9 zOv?b%GQhM9Ff9X2OCNd-FfC3SU|JqVc0VGRmIn=HFa=Q-thCGlU6-+OIvJyBK zU2nT@G^50swX+y>8v=7lfhf0uxPaT4uWrJ($pVqR6S-YFJz#R}fd_MvG}1nfNY`|= zMuU%uvjaxHooL|bAEAIm2#K)KRjN#;n|rcI?7Yd|Q%QA}DEYq`Z*IW3EU(_OY*3yIIFoQC!kgvY9j+AH#R zXJ?Dw{$&^)d5+qHAeCI)1gn^dI8Ff9_S*_pjAwUg2;B!5hu%A?Dg{e)(7!eTIT9ub z(#4YC{eTp-7AC&<&SxJpA>}`pJ9AY@esI*(9V(StaWJXDQkq3$WsgLQ z6ST3?vcyubinM5FF3{4=NFK2zg)iX{JJaP4&gDO93`!#E`tGOCr*y)7_1>(7$Y+HuYh zz_v0BQ=mlbucy%7ul?OoasugHdeJXDt5?u=u8H?EGp@G*H-!_U?l8QHA32Yqy33aZ za}Zb`vIB?ZeRK&33@~FgVk7*8^An=SvaYh!U)9EOUAfH9J#fxLY{XMLy%>y%x6wSw z#t(GTY9~8+;{ehFVkg5wr?S%T9y?6BX&?!Cz=Ub3Ra2>myKw{>wjqI2)L(}9(CbEy zLQS&1U-_63Ml+pDR78O`k`Fyop5A^%?FU}@VP9XsIB%h3efu341@R z347Ykd-zg%%On@yCM$8LJ)FaW3J8G{!h?bef;bObn;U)hkiLsskiLr_)$)m_F?IBQ z-RcC^x!uj(J>)uOP*#g~(7R`Q0nI4LIS{Ck!ToHI zOasVcV(ugT)DG!_l+c>3S?88&HLL#FqIrKkWY(Z+W)mfk8ts@!8U=SsClM>IhqJhn z4&R!NIqnL!V~4xJF8dzuCASs1YV=>U$g4J8YVdWDYFI34&Z= zzu6}XdrwyqxKUc`WzYSxM-tQC4GNlfWB))9+g!2djY}L8gVCGM`jC|aR_G_(nd%#x zKcjT()3=3?j?olk`w3GEo;g`cod@7S?U(NA^u9dQ69i65(0F0dAnhE&Jp-&lClBOF zZ5&F~5lRh(alMt_+d@UKQcFkvt{1`fH~|61*}d()2p0cj@)_%m^9idoCPDC7K_9p! zutYoJ->Sz;4nb;7@RHHDS28%!TXFwwA`aVz9kd8LZt^|-`M1esOJVoPkW<%<*@_{F zg?QO4Aq8`AMA#tXNb9?r6?IJ|kUq93Mv#JG*%|nxHe{O}OJG~}JSu5DLh`+nqZrYv zAK_MQbN*gEbz|uh=*nmKnMTrsvf|XLmD@3O!kKuzd$6<1&7Q=vp`+0yS^lYdkvv^) z<=K5`rzEcnTpaSv8(4B@vwweXY#~v!G1MX}qxWiO6}))V13ci`+hm4iNl#JGJnM_N z9;XX)^`c&K53F@Tt@9ml&b*-f=p$$B6%Pa(GoJE?1OP zo+Z9&+W%3bb!)$MYFqCv!zD3S1%{SvGGRu+b}GYmscnd{mAlkqTG={U8CAkV#MwF9 zUaf#=%JS404KKqK0*{iukCG2F`TTPT7(E_i&2~=t_cOcG>n?7G^Eu$~{ZiVO(5NRF z(NTO)DtB_#2(+4wvZb9B6;H?}kG*CVN<6a@vGK?GrM<|>w+qyy_|?S<<6phvK}Ob~ zRuw-+RHGVFp|46A$l`eLJvYtgXQunnKZw@9`3IB@!W*-IE)9(DzEr%0lL2D9@f*Bv zx|u^hii!jE)d$@Z_Jqpl_p|(8@sz>1QYT2~(cs-0WtKn%NmRE)28~_!*L)Q{B6D4|XwT zeZud31u|lE{lw1t_~{t6>v-M;?i#jxAHQ>7o%2OS`0f?^?j@gsLF+BB{bmlPRf)pwT#nG zCu_WkiPJ&DhZ8kAou$ur2FjU7n#=_^Fadv6Vbp>JjjFBJ?)p+ev7fzxl_(Kl56v+d zE^2trO)@RC5+)_wIi6l&EUZ+0U&ZX;$bIa!X+)aL5yU;&0|8V7g_1gQ@^JmF^zr3XKKZoeSGeAEDsYRCn7s8e{dN^H7_dM0utH33X0Y>BOKo2&A9wHqtYiPCLoleP<#C?RJh&qqN7%N!#+t6NM9@Bm&Osx?kh?Lu9PS>6>9p&bQ>3f$KI2u3+`LF*WPe)y7j%5Vldk%z|Hvw)MP-;` zv24=&d@wQYr_tU<(@6;5IELQuSG)eA8!X%W+f98?=zykjWM>kzUnIq}BeR7hd4Gif z6=Hns53{OPBNA@Eu-{q;<&K&_ot-DBL&ri`5lz~&|JBe7TYl-0rY2%J|EOsrp6Y4m zto!mMSLcBUH>{Is4$m7I9hj9K^c&q4|TvfThGoLEb7{gqijVLQG=IgPmSi&XKd^5apA!! z&fwm4>|6C%Beep~L*6DMohgvlhr+fLsLtdW;|qwdc5!V~HdpD(PMLyA_% zs7a2j`;uMNFA7S`D_4Irm0{CMzS{0Rw!+9Ja5KvUZGvHUIE!8yGtF9VuRy1t+}UQD z8zA+|_jBO6SM{ioBoGPQy-u-?K1Y1|(fRXkkncBb-7JQdV>KDG0b@(0<+ahTtGbP@jc@5I!CX}*^eYsdt#rS3_#CpIGW8WFLD8%dUKw=-c87G_UD!_ zqa)!WMhGZ(6BE9QxA+_TG!^qZvQ;zI_l(5OWt6t)Sk<)mn*A@I@yK#RZpe2bX_D5m znRCv(2%MktO)dqIS`tdr(JRm{mNfp@M3HS4b|AqGcN2<3;WTiJl%n?17jM}KbdXyt zUpI5Iod7N0TwXrAF*KmRTHP$gH?Oc!KQ_jfEA#6*l^@?erkBpIrxuvC;;XmftGDc% z^1M5FSR3m;dm#F|-gN8vcP&k&`km2k95GYw4Sl~xqcI(rqP%ZC)G115o%Bun7dYOV zU7|HMy%(?otqGnNHLE-9Bnm`^FdR-Vh>?ByjnR1uON%G;cWk|v1w3tyk2^dq-6odS z74F4Y&W0^KZB1)z6nk{KvDkCX&TG8icg)*q+0OE2+0L`8o64=&3rmIj93RsTi&+dd z^Jp5ISz^u~LiO^%lq=PvhuG(A?B ze^u$HQT0`0EdJ4H;MSiS0Nm_QOb8|4ti;msQD;vv;Ojuq^?rr?X+wq!#3Ho3MS-gl zCD27~bL;$RT9pytcwSJ4Tj53!){vILo?esO zFh8G)b2?Cza#I!a#@g^8B^k>scwc}jw7F6phn|s_C|^@dho*)jB<-J_=tvIqm-t%( z&QoSnOfA2@Jl+pMoe2TLEfGp2^E%uf7($7!l+d6GXs}KTid1y_oqd(}S3Q)egQ3vg z{xvntTfxGbIxPFmJyT+kOad&A z#x&B>XU^Wq2sLO==pI9jC7>~KQDxF$&jm+1LzPTkWB!_LC7E zXXBc7##Noefd7;T#?`GfYr82{_LB@8=jmr{um1tWaGa-`zj@X`?d^CN$Jc8s$tOGA zZ)ET6YqfE+T`}uswthZk>gWVLT@DFC6$B~U9I`V4ZCBH=!cgZD&DrvPv&eZx0TVr6#K2CSQXBUXe^Do z&`OzLs*9-`q-lBM9AOfFszYI{Reh~Lfs9$sIo0m!>Y}F{E5PkvMe(mN%AmXg9c?s} z+8s+n+S#hq(jqP3**=?Wf@5-V-_@A0NrVyFnf6-R?U#2RqhNNHmNTrmT2+F*UaGRq zGbPQK@)NnsCmH!v$$b~$?b7;_&Bswo;@!)je2MvDqc$+CLY^Uf<@WDByJ?8p1h&Lt)piBfh}=4^fSL$}p6h4?YIGi5jU_@_2%ziNLi|&fuS)g-eg}DT zVD8;~Fh#Z3FlmlE3UUD~cP$+;vx)uFvT3Jk-%BYr?YqX?ZgJ8LI${JZ6WK;-vztjW zptJzN>_DDnYN~~59s{+%FDwV?pb(qjKJ3eU!S%Rsv%y49yVrUaC(C>#1TaQ!K;Sf0 zZ2Z`4w!Iv`ILF#L)P;BNN?hAC8OlB?0W|cBT&Wff2YJ;J+il|kxemt1k;g_1bPux# zjTXMxhK5>f?s86gmf&LJT?n)62;n_q!uvNVKj`4=qQH!y^4E|eiX(%8p@NI}WdFvS z#DB?%&uIVE0608piwup#ttD_2xnPlBi%W_d7P$*bgc)t%5ocU8#@-AXQ@DlaVPW@= z_CdS-OMl~a2>7-B-SjIq2iZVHK&lChvGbjVXQn~gv0L!o^!hf{foCxmn1Mo~W}Ae2Kf4ep&&f&Uf8 zDJNft9387?H|`EP*Wz?7sc(E6K^LX`jw*E();u_#L8s`n*$He3i2R}H5T0hv^lxtlhe0t_Kj*Y3hntyQg z*L)$598%Th>+-3p7Rzk)_q9&wHIOSnpAxM zhm0uXCvy!ph;(R%Sf0PjBX1Bg?OcJ#iNnCH3qx zim$Ka6*1N*v}a=t9|}X;?hlK0{CYfL1SB>6HEq}X1|!6@pV8Ux#VO|>znKokg~n}d z4Mqd1VRqhs#A$e%iplTmyZ!c@5~zbOeNJ6dvJ`43Wwb|ra-WfQtrls`eV?eu=vmfj zo?|6)wn3P%6f*`bFy>}ts;ypu%(wb5ClN>)6$yP_fkTp29We(=vkxcsqq>kn7ug$- z)nw6OWeb_+|$qqtzdRx3;vE07q0oGCcSgKLx{NhfH0e z>gR@>y}8Om`mkb$cDr42yKvgR)%J+lG0lOj{?0Q|uRyA7Cl#mb7;0)ek|y!HZL z+T0S#UJs_Y#;(@^Ug~>ITG3wBX02vP-NMIJM5AC6JB4LFyOUQ?Cr`U3>|WE4!9s;K zv*HtMLF&as&xv(8Z87{yOw&iPrG%gP{<%FXljpbB%s)P|+%II>;F9QZazFiX^8I{` z9Ot_(H!w@ptHod$w?7K8-H+m=QgtL~6Gc7V)E#VsT>SW7sV6o3_-EuSK_ks)L}-(T z#hX)|_xj*)L?4TGdAz25G^9dBTa_dd+9&;If)A=_pM+ywB}F^ATX3qPI~iSr+OHF~ zv^X0r&{f`=Ap7DId@kc$`1&{@+Z0!LY<`7xm%tTbm`(*)!=`EbEo!ercGJL}_}w(N zhs^k0m*}&?Q+C5)6!aw=U-30|mIkf+0N;h8s$i{)3{a&E5oWiplCNh(xTRyAIB@C& zJ_v*c+obyD9(Q?H6`aCeJ27oZyWUgNKA@Bn+R{-vy*WuYWx4^qqF8vuNDhj*4Tw*B zQ=3Kc`ro9`LyEdd86n9mP%@hq$KmD82+>m9%N1$oGr)*Mx__`I`Z<$EgNWm>VMkvW ziR=rVWWfowSRha6Q+UE(MkQhrM(>3KMbKfG#Zn-5@x$!F!|3kHPMau<+1^p(TI~|3 zRBvZ{`xkr_2+$pfD7Y_wdy~1&IgO5@4zZZU7Bd(mJmwL&_h&}=ya5OUI)ZOFmQT#| zOcK#t%tRr)Ymk9b=IW?R0<+ZWj|FgoCsFQy%qpTgl;toLI7YKM&YK6a61Sv}Ol1`D zuGh=rjFGc=jd92wm*-#u{Su>QBpW1|oIc9G&3lFt-i{mG$30T+KdBRx@-^j;Zmhdi z->ELiyRh|kX2)o5`sP^5Yxw+L5n;rEAULQ505DXn6}Lm%>uqf?baZR$JZ8M+8I{eF z^{X&C+cQW~o=FQS8Po(VfFt%;AQtBkYzS7nzxJAFr255x?#t%&pG2hss_VA#&a6!v zaW6xcMwp%ph4~%96s)KWglD6j)G}HAwz0P=nTZ|gRfJM~ zs83p@$0f|c)Lcxm>xoqNUSmZ>`_`S!1|RkIdoOc2^thMVl4NHIs8qx`vg1-4!J^d+ zCKC0Dx~2!b>mnH@7MAv|Ff-=sFZ(p3ohA;rWd7|g?ZC34$D4g|310h+Z?w707UMr3 zzm~6?+&W1L*{4$zcA6B70Ix~>yxm?CiKg4U2a=69c=tzM?y(-IJ)PoTzl zfa6{qD0NO8cFs zLNNE+&iPGmC8B?vz$hPNHyIUu{RgoK*(h{<{f#)%dn_3#pJn05SJ&*^B^vCt zY$|{sae+9*MAyEfZ3l_BP$Md;5?Bd={4@dhanFFcCy2xccbT2IrLTba5t18xww!R1 zZ(QL&NfZ2``dFDCeZk-P<5Z`-eARybCLHIzmI0eVmfXL8L3hkeMf_EOB#?2GNdL9y zM*rJDt7(u)g`@OdEaBuz#YTm&M@2=nzREeu*brMuW(8$!_{(bh+@!u*El4E!oc&y-oi^!0tdn;Uiim??bMTLs*n@oceC zepMu0_M(Ue`w8Q;ff|~cL4xsp?_U(B#)`^v9Q8yXVe@yf(r>XbjdMgof0b9MpW!QW z!bjdeAd(Ik{b`qCgcLkmMIyp01bA5MwFBClXZUrxmnzxFCp4hJ;1KehXra6t_p5!@ z@G9KJ1`O298(?_$lBW`##JbhEXXU25M*(Z`h&RO`EfTwYR#(W(zq~A@-VNIPcZe{^ z*V@NaVvGmFXm1sz47`vf zb6rZubg!q);%+>LRRXIN5?n%DQe7}XbxWSMb8F9$&~v8hpfTFl z_Q;^;M*P&B2h?f+M$eL8$o6pkW-8PuK$)g#45dqhrj>Rb z*oXnP_LSm>ABxD+hN0Ld5cKm%(=Z6JqS#`3Kyn)Sl^#HMTVRPyhpulZik4*xRDvii zuR@wK^ZtJhmhz)V#YakY+4+|%d0QUbgwzSyh+2i^XcFqQboDOl%)mr00@`#d^sviekP?D)Tpmgu-i6sL8o z{{rkeZ*g8|MC#o8*)YY&q^wh`7e!< z5i1oS&{r|~b8LpczihUG7(=HiQ)$bI7}R$EeNA9x+KQ@n-}ip)gjqxE^1C{u?$0*3 z`pfPj>VabE)r>^Uqb_12u1KCGS>dTpyRmDYl8M3+wgvAMnXlFoA^r_O0{ zJKSamf#Gpee_QXN>4#_YA1rf25lwX`tPCDG!4ZQ1Vp4(-UFdhUDGN63Q3OX)XlQfe zb0EtfU;Bxr;K+#dD6RP7YI@U!HI72WVUK52(c&kyy&xA6g^FA;twwoKl^4B;PwAvArZgTP4SV; zgSp+c)bQWUsTZh~#X+om+)&rkq$0L@n9*$4trqYn?uwg=uatVJtgN+{A#E7+*295Z znNHLB2zGW2KMNd%16)*9g(6rOYAjS$o3<(%-NLx*%AL`$*4;iPU6gW$q(awK4P?n3 zVJ9ran;q5?>db)aon*BM5=mIWnp=x+;x+lg(buh#U#%ri9aL~S)Me!(*Ja!do@Y(g zHD7|$AGquAQC{0A!McZ$iao?53}P)Y@zqF~y^Iab{}G9GD;>Ssuq5pF{uL(%UuLjB z$4behgD<1^fyC;dkF$smLuY|Ns&h7nleCVZ6N0I0{O(t0{#`yl zpYucPJYPb+d?n5T;y-G#2M=Hy${3S4(+NTj%mW3VMRnIC3!m8P9X zQSR<)xrH81OVTEz4TJNmy(Nc+2K<>i3MN9xmgmSr6*NiAR!Etw^27V=wobZh>~#7{L!i@UH`eF zkz0bE41C=+ieopY^pc(Up5B|^0c^kOb|)pzusm$F_`)UM6rJOJW&``Dd`K1*QcgP| z9ak`1=Q_62N3GlmiJPQ^8c3z*uHkUvcgk_ssDsMIOs%EQE$1<-sz-sVyi))=&uEmd z)Op-1dh-lN&SYu0>usQZkhq!GRMEX6+$34?v|I5k?S@L~VIH4b9+W`#H%~a6l%Hwo)dZ+$n z3{%6NlznTAXDh~A9qR;M}pXLmz6HO)i^ieJbt%(+n5AIq|=II@8)r+?rS?IYiG)Wn&yk z0W8)|b8MTI<$wkPMX^sYiFY<@(~x_nV1ZTZ?p_CKfXmnPLA#Wq`@a&?kE^qvM*iJg zLWPy5IX7u`>WN)L{Zz|pxhdpiwkI$e)C|H~#}y#wMfUj<4LZ4&%Dpy+BVIY{?)9hK z4UU5CW_rH{8(rG3w^ao00DT%YZ$aCfb}I#)`f|=enLF{*`I11eeJ|;ZLb(woR(LXU zKKjBJKk%?{V*8g}2yVgJ_JtqM`MTb#kj|9dNE2C8*AFwTXZ8zb^`9ia37l;P zT>*CK3oeP`NIqF>*l1LZc+p?ZjYLL*3N`Op_A0l`_6NeS#_|)EyX*6N*Z9!j`U=1} z^uT`X%@om-#n9YZhZ^_qpZP0nw)r1X&JrQHdCr6++J+vWzTY5M*X0naE>*l<2A7wNv2ImGa7Xm_z${IIu^rMSp8fPU*;%b?*U*^VpC&gS-(obb|i*NY9%ZpYZI zb@jHUMl0ycSbwY&d$(vwjc=W7aU*0CoebIfL-JO84@1s6K3~Z(0CVX31PR|>Iz@xR zGb0@R+rA(r&5`n76g21do@z^&eO7jMw$Jl%>PfF3;}7rW72MlxB7HB{=LLQ4&;B^3 zzR!IsxqJ`D=Om^d$LH16BYn>Y7}w@tYta8!=7p+2d_hIAZZjRZmF(-?VkXO%A9A=fy{J+f0?*B3`jJ^}f0Z!6ebVCBRQg=Uy zquGyfa`&WS`T0g6s-6xaGWuEP=`n%Pw;@vW8kkSMzi01^=ITuSn8*9{PR=d?obQ=ctp36G~L6mThz4L zDV9rMF4An?BGa>Q&W4h2z`xY=XL{{_j!B#2@_4AhXEJozZS2QGDl~!Qar?`0OA{*b zhVvfRXqgXNM@YXmdsA@TYWydP@)3jTaTJNuAv79vwvfv=04vRu~ngx+$?Mv8W+HlXWf7awQ@PdPJ`G-wc*x||a z=#UR9+(D+5qkVN&{F32dC2^>B`&&9^x?0$f1ah*Ogu!QFxO=yxT8?Tl(l>e0NPX}xj}@8E^UzNK z6Oy&TwQ3NBR?e`2;WMzqAw06<878F(bYvPN(iIW~u!3@qa^!QaW~j>53-a4~iD9^a z+m&vR-~e-oCW$(;xs2kn3=62V9)P2dvaNGB*7Z7jYrzDaI&DA>!^DY$T3HpHmZ{Yn zZ2UG`3T7jQ>Oyi|TpVdn#f0Y3heL_7OaB~m7tuO1&3GZbe~u;RJQ_|bxT|du@NUI+ zof6XL(loD!En^v^C(An{o>Wg+@8rVr@|UV~gAf(b$BC8ZlmJrI%~i8yrm44E$G`_r zw^Sk4DNvJ(@v4?H8c234Wy^u78*!N#`ZF>}C*Dp=rA-`Cr-43lKBuc(%$lg`GPrco zuqYE){F?{N70aRE?9X6W3X3eaqW2>_{lyAQcPWmZN@D{hcxX{|mk=b`j_L3^vS}6X zavjU_Uum%2tht0kSHy}SlVm-{c_tV|%KioTOdGHxN@f%Yu_@c=!XOHN930&`9eg?G zK+<~n?PxGFg2IT4>J|mRwI@yHmZ>v^1Pb=@WC$(Egy~q<+S}CZ$RxO)5=t8^HN=Lm za1@CXGyv%yz3l^&>51Jm*fa$B%QN%R^t1XD}=Rdq15>D#+PO+KhcrygQ&C1 z5iIkr+oH44%s_xBG6&1D8Ii<@BbVd)4&n#=6^f}xV2Kg3xnqnFd$QV+@F_okB7uX{ z4``MnFD=@PcQeJrpgC-#4B3h=iz-vLEOxF5PkZKK)Zbe+rd(!&g;Z8oaTcze!Rd~{ z@wK$RNYCWnX{-k493iJ(Qb{=fBeE<>Qikg%@K1K)LrTuu>oGO+92^eZ_&X!cV1Rj^ ziX_1U^(k-4U7dalcr6dq?4s-Yek8=|ctY#!yj9^T88nKl1sjAi^H`Z*8F$V;nh7BoWVlp0YCxOlk@u#n$WdC-Y3Ge;$ zE>%gMQoGb)LZTxYCl&OU>U2wZ0nqzgN~rKF?&U*FqUg5{HL%@F2veF=9yvRq9OaVZ zd{#W%N>>x5Kwy(1=-G<9CdProX`?BuYS~ME;&5k2Ivef;ZjOv99WOU8xb{KlTy>r} z$F`{rWzFid1PevR(~Nr)n{`i%b^s(;4;J*XPv#(Ck?kqWW)5sBMb`DESXyBqgn;Q3 z-lB!q@~4o*og{Yh_$f|jVR)gsQCe?vixarBlULhHDVK(UMSA=pryx>AWcDjP+JteO zfv=;%QP~VIkIfY*E47>-EX=n5UcZat7o(y0fFM7Le0bctPpnswXDx}h9Ja=>4vWu0 zf%mX=RR_B!JakcXvl_WZS;E?yoBrTaG8sOo=NWdPpgv6~{szdjZ2^G|TPQA#^`NFY z_cAZ_;)dI#_(x;~+6wxsS|ULYOC4ONay|44e359eG_9o=%L7?=UL=0d7vfHLnvEhd z%#v}f*323M?;f-dG71xvbiknw1ijVYm|WHb>q9MD#H6IQHC6Gw`N}_(UJT26#l}`~ zHI^Nx0h!iC1L?zNOzoz>yt&>2OhPPwC2$#gqdLU!(tpCI@+-rsv-|2S?ma4^??pJj zBSRHZUdyoGA$k@ddVz?Av9#a&% z(o~wp{eMhkK0E7~TdK`#G@e?L**K`R?j0j=?D|;D3u(d4-~w}Ga z=`|AP)GSv^23|ZY*{RgkLJUta8DcLR!s{`kFI%a+Mp5cCqbF%tJBskCV@3kHQc`C1*nHLo zy4N&HyXM`rk4><2V`Wg5n(M7rP=D*8Zs0?g&h%)Edw{l$pKVy5zb@VG4_){gK@V9$ z$9A$MEP z5=C>POF>A6rb|@^Oku1)aSwn!ktS4=SB()!RD_}_yf%8*NZEq!Z8x2wd`Ko1lINGc zes(n1*NiDQ4p1b??e6V^M9^HZ~@$X=_dcpkd*JlT9^glMSLP0i4 z7Hzt)2(7aDxMFxR20h!MMJ%_MdwHz58819~9qfyRP^kIhi3lVzw=&UqvOIJlTz~p# zAjwr@OL&?(#@#l@xp07QH*V@n99+AJ(+>2nl_QsOL806E^v(C~aoj*aCt_ZGfQEbM zfO|0E1bUc^2n#8ZhIOx13v{Pu_dtsO3`Z}uQ>*sGQfLyXH-=Zkq@h%|=XFa?nn)B| zME5tzSQdxrgi4=}X0OdI5zRd9*%CZ1$o=7_RmXuhxWUOG1jC;)Rcl&_2IH1PikB)0 zO|VtjqH$6T-fZzSN`Q`*ND1`IZ=Eb9J=Kk3N5pD3@reb{8CkQc`I8#!${0};HtVYi z(9GBVD!W@gXF9Vueg%4T7tWP7#zU)hKCO&q)|VmlZ1aTC`>o&AMKS~BvlqaGI`2ok6}}|(R?4WWkNI`h!;10xLWz}%;bF%sii#XjY|a-wj+Ye>FRaBKBKC*t zn-{{z>+piLP1++4Mzwjf_>;C`s{1hH->S@IDUMoognO@!U$X#qIJHSjU25;^e|q9W zIBL}!VJphE4SNZwJqn+{cyP#731q4o$Qqk)In)%VjgWr+8Ia8f466T~_&3qtBt^mj zedGD$_kHVl{?F<+Dt@+NA+wc>SjgsCR3IByp_uv7S=66w#hU1uvUzM)uEGpk7qRQx z`!LSJ3hS9YX(!-_<@`no-ETFPh~DXC`lS={pJ>jT(5)_2D7z^|lFQb8$d|S*`qi6l zNB6^)Y@?2_Nk`A&or0TQc0ZOALR05XZ+v%tOThwLiO^~%c4Ad@@qaN{jLT3eH5t|0 zO9XxB*-0u?X$~`sZL%w4s4KsIQWQx{4Mdu`53p{53E?9i7RH}xO6_tfMNLnV^D>Ki4x3%Db*jNOs2VhympO7&Twt+1xf%&Y1N7# zP`$fE;fp4wA8GBpeq!CEFH00I5|?Rx`<6FR7q0s!p0ptnvNd-7N(QHKeSKxhOr3KL z)bg4Ky)xLn-tu~Gw^RD1y+nY=pS)!ZeY%kfebR-V`FCJ@^kF3Ldq>!shur1pXo{eU zh{WC=+YN$wx}@%D{#g_(S$NzHz0rlI=1}XyoU}_3gl$WGkM$_9tIAv(fb+PW!6Cbp zh`i1KtZ)MX_C= z@`d1Z=~Eu#$Wg+lk2A=C{Q5)-unXh=awNCWOHDZtZPMEk6v+|?YjM~2xh-BSZ1GDr z_R?Z13hnVIEFRmuuF%|_e75fyfPOkd%;APNU5<7=#~71gA?;SgOuj*1*ZII^px&Db zFE9z-1e3l`5X(hr^EqflTTs()@j6pe(F!iYi&QA`|e*^FI+1=4*df)YvTW-!a^ z{z$Nb*Tb1Ug~yxlW*}o!{2TlnY|Q&sk$*7mkd%r?!d<`YUP9EupCeiw3eO)E|L^)E zg(7b(_WWag<6JdGPJE;E=}{Mb5@0udCOv{{k_qX1bi#T23g{VRKMMA*PPXY1;m1D)}qU( zmDO#+$nA(F8b2>E{;fkNY}1w;wlY(|hwpo2y+sGq<2aVdFGYMk4clVHnFgOTvY8bk zHo?maQRo#SG)HkTq16A;cEl2FK#E=r-|BZX{N-4W?bZJ(j^1kc??!L0r03A9V(aM% zx0X{@QIA5q$5@~^6YpQ|;Rk!P=^0JnA>l?H10rSMVSf4Xj0QNl0(hzY@`w}%keXF( z0GoRMia~`%ZQ#C{@m)h4Cowj9e01c<*!ZEojg1Wd($e^$p7CAdire_1eoQ`Ud=%7H zhRL-TLCahm|Ir^6na{Xv#sB|_s$k~-J~1lUfoJv$ zk$RuS$OkaF{|2dvmIOAj??J((FH`nc&VH4*snzNWzAWO9aI-9hxr{6(%tDpcTtW8l z?No;qr?B;bM>yM{G34^ly#j6JaTKfLNCtFu)o_rC;x<#DVJ^6JTE#zaT| zOIS}PxBS1(s#vPv-|=}f(VHRTc^C_pKI`6$HCDplu!X_EEksE%NP1gT3GO%wv;^g&rLzl`-j1qXVH?#+D>O*Bf40XYzKkO#)-5Oz(~SKb5%VC zk`A)Tqi@#WoBdUNG=vTfMP4l%Yy?h~xz0JZN*57Nz z2-)TiOFI6duPe-MQnU=<2SlWx$AM0Q%VvN!(M5apt7bwwMi5#>>GxFV0;`&-_Pt6< zq1vk9f?2Xo$iXz9&zL6;bTB44=DTn@{GO3YCLu7`cjS_iGmJ2UY|h%>tsTKhk(mxD zaXRD5J!D3*sieS3PNp{~2Ftry~xlRt=~VwW~^l)R#=W~xmYr6e>hjr^Ll z(~G6pr`YEVnXi?E+b@{4DGWy~(51E^*m}r>)9o-0v}g>=W|KN+y9@~wI6tc7Jfdd^ zlO0dET_Y@mja4-1kWNlMY(7qGyg`j{s!c2*9c!JoE?!n23&nyiQr-7iv8P||J~M+I zSNXih{5suD4x}ks5KZVU%lqGpKyjJ&ls0q_6-H;-uZyQHn@T8%KAV67Rd5Q}Z} zI6{K7`hw!YOh#=6#6S+zfZN?jT9%X8T*iSOlyYn~7iF;7dh<>iHP`Q>@=g``JtZGF z{T@~)M?jY&miajH@%o85!L-McF)4lSU?M@BIWTAQ`c3DtW6ff6~@ zk^m)jJ5(hQsz)JBxhructCZV zYP77*`qdJIaRlYJZDnQjx4qbAe%v>&Ot_j(lb8~0UC3rPW*3E0CQ2Iq5{JU078a6i#c zFxyecH3ynJ#%O@EIpAX{_a=8`^5$XI+ShzLzMmXC%f{h6B_KLPVA;*}CIZ?;(r}(o zuMbffxAOe-pPVBo?B;ei@#s)y5df0*&Jl)Nd45b*I!4IsXT@)yhp55Tg&afDm)W(r z$xb)drZrJ4Wl6`r!v@x#ZkYjM`1*Cvv277_CfT@RFCej-2yNSqcLE0ng}{-dnSdo* zDqUbM7n(w!Zp3_JE9R_zS5N!f$-!{)Zaen*26)(=fBs*@{mp=V6&+qo5$i{$%_O<2 zn`rDR{l&G~-+oo`?HiUdDzVA&j}7JpXaDvy2$^VL3AjKDl&1HxT#W|! zK^+JUGL<$u3kLb3OV&~~^Edi+BDjKyBwV}yZrbc)ec(phpgq%}c=d!2oNFS~3wL|; zBHf_HBg6|s#0P)&l}K@3;~jjTGaL?=8j219;d5P?O!Mg`g1!74QNa&RJmCicx(C`# z>Swg=f0H=cs%ntcp9tPA%kF!Cfz4lydVps>D2+2JQMB$DLpXu zdP1`Pe}w#FlP?Vy^b59a+qP}nwtKg2?Y3Dz?4Uq7s~euKAvC>JBQ1+H&7Dgs{h2-uC z?vs~ubZw^wZl_WK+Mlx+81LH_M;r z%LT=w5ynKZxcI&wQ?*x|oJ1D(eG`+iA8Hj65~f0vgsz-)T1s4+6D^r0$@L3SvBb;y zaMDP)En7oQcUXfah~xFe)%!xz+s7=zz>{dcXI7B35iD23uGrCFqt9GiM4a*A@rKy6 z;BbEmzE)_w(Z7iEpQ%uU1#|UV=a8XqE5OaF`@B|d6F z4C>PQU~?_8i>8_6h_3J`gRt=NjSornK63(JTPvjT0Yr&vT2xe^;7+0v?%I@W=H}2d8ey3Z>M|_KE-D@SqPOV1my|luEFywg^KP8!tbd7njci15i<_BlYyx^)AF|;$ z;*Wb*Hs^Whtqt@N@{W(uAQxNjz04u6cm14Tgw7+OXxMd=`Wi<#$ba&D$(%qwL_;pt zHc2D4xSV)}mPexjz%TDelvSv)s8#nQ>^fi9G#}2wC(FU?BDzG{8UWNkG8u~=W`FHY zgN{NLox=_2)p&G`a@I;7W-%&Dl>>^XcdkR)NFxJCdWJB;w30XEsvLI(HT|Goq@5X z!R6cTnhNAEP*a_3O(ekPSV%s)vKM%R@{;)uakFhAO3!qROBLtYmA?&|_IAPv^f9#E zCs?zdaKy8!+<>G6UTi*P;Qga~e?hpW%$)e2)xa|hR&P_H;b~Tqj%lLX67K2Ka zPO9{?suJr&;=b((VljhZ&XP%6!Jzjsy-{-RUm=nxI@YVS@mq+h?e_FMHxNeBNHZrB zcWoj0*Q;fY8*&Omv!7~u=PW8Oiqr&d%QH-FVI={UWts)J)IBt*h|p5Ts80R!rQjI7 z{5nszKd9bOBGW0TP6;6<4yGxrE|6EL+Z03Tfr2i#Fbx;<$eVJfo;UCldVQTAq2KEN zeps?aiTZ8s^Aj6$#a8Cr$esdj#0|9t;G zEs%0DB%TO6+E|`pt2zfrTGOjFkeRTIGNN)_EhmdKGN=Q)-ieM<%DDuL!Bl6z4T+2n zW{9h}f)M}5+cPLaCT@ZMXnSpmTbxtRre}d?(7!9eiEiSVyQJN)a_mj1y}Je2{n!LEL}Bdh-?sf z=3iQ}f(E;Ih1DXyCkdq-&^~tn@D&z;AohE288<`f(QFTX*58dwH5&n(-A73|!CXlY zmbU$jC{Y10YGlr7=(ut+6Z|$m2zv3^?{xOHqClM?W1V1O9sC9lkOmWW(jpK$Sao@` zoZ6P-U0*vX9@2bNbpA-`)=kvy;6C5?<5NQgc}{%t>G_fO%g44gL4n}CiVSr>rdF7} z8Re})tM8DtC#h!V?%V@f4AAoDT#J&_omHQmxdu6N5L`1u@A7G=2}rBlwU5}=DKH3(1}w4l5K>`WKo`Ivr`)RS00>$)VeoYJAxdYi|*E_}`KRg{<&L*oMG?6JS> zf%xCY(PuLlW8DWH+4$SN962%%HhEj{-4}dooAk`B4cV6IVL=9wYYz4_Sgww7^{d>5 zJfHI(tDR*IMIL0D=645hlpe2IB}OfSQL}$A)KPCCP?(K{kq#kvEfK);+hDBKTc;zs zF#4K40C-Dq)5YMdR>44YNk^rbOeqA_I|Sdo9&e)^)!udfiE9y2*Q>Su%uH>&nH3`;E+ic z@T6ZPJ$4kR_4T+AP0F3wp^3jv!JdK7bdG6umAQLHiIpxm-E5btA#pGe{0h=xoDjrZ z<#2S^A_RhsQ__r-Uo&+X>o5%ZvT!>PLCMEMTTR9BmDa`a)=$c@0+kDr>>m{pgz8oQ z%`NPm{|Og%|Cxy6m2Cf8+?xNd;L2ICb?a^>2lGdTfC-`c9rEc}$Fj$Edqj;aQ~lEL zN61?@^1bQ!3JmcxLV--%-hp2AY54=Yn*`kZkn+LI7lW-r-pSUNq4`mr_G>J9K|J#? zQ+I-PKmf(J)(fuHWB?rNj}WSV$ttE*Z(a6@>5=`o>n_VwID6=yCchan!V({VK&l#v9@{#>}n#kNg+rTb;mdP#6|gvQv_)p+xc%A zmF{zIUrl?}Vau7JuTQObXXu1E@nF@<>}gi)Ox?W(NXqaEcpUBNj?Y6rc-s&H(Xig$rk zy;{j8wo{)C@ufv%(2a#x8Z+ckdwEu}r@waT4x5(H)1shi*tPQ5rPJhk^Md_`Al|xQl8yHPCsH`!CA9b!FsPfK?F}fhNFAu)_Mg9Co^D*&Wlmd1k%lnii&_@6+N&7Qy>*p&n z6$So`OI26gdch9wr2mwiA>U`%`Djde&S>QYNoXHp=qYH@$T`gGAz@)$hINA ze<~URfj=o-giUAw$Jo^)AFJNo&`9THj+Kf1(k99SePqS=wjz$RuK>_WIvioBq}JdEKKcI z=GQriyR68Gzmi{&MKPq-LW$NrRch&aj-5p>SV?+`x0cyC9wSDS;gU3 z6716wJ)}Zjv*IF_00?>~jg#21GX~du&iwo$o1&Qa#fZC@CU7{c?D#|vZfUk{GSUom zul9T$13OP*J;CzQz~)L!*{0<2Mur`DEo`_Fj5-Is9|b|al2y!fXZ2O*yVIaLE9>ey zd+5TH!gQ*m4Ms2EL)*b|kYH=$%o0HI3GOKVsLFUR;bO4YP6Uy@s;aV5UHzt*WDF^S zQdy943pDn^aibE|@$1#PPC$hi)=b18dqJW(3kq_WrFMi9XzN~?OT&DQkvdcmhF0Q$ zPIRWvFGoRRwX`0SbT^C;9UFtnN+InaY)cz&(!YLvU@|tp-=95kSfh{Oq${=A-)lx0 z;eRJhxi4vZb;1f$l_`Ire1ASZuBRSytfn=xZQ)ZFwqL})!x8&>a{=_pv`J`7wbLy6pYIpO`p8xkAc44zGk9rfXq)o zO!D90%C7?^2syUEjsD!Ma7X3z4c|nO{`Yo)<+SEX38w;RykQ<1o#%Q#Q(02+#V3m&u z)aB#hX!*o<9RoFPaxRNYS7I;iJ^r}$Uka*F!;&cG=X^gmG4gxg6{7`G3!Co=)D@02 zWNS9{eh_(OgqGMQuDqJ{nUl97$*t4?j}Xu(+}6^+tl#OF^SUBChXZo|9J_g)*0=0E zz*sp3&CUlDc$^9Dp*&*tt)d{pNA%og7R`d5oZfzJOl6+fk3{6IH$o1o`8%ryo8xL%faOh{0N$6&{S2o1)(|2T7 z;x!4K&Pf^khD+nypODi94i+W#J|biZoz4kyYORJ#Utb;nvuZg0kJDKf`+rjQoBt)g z=VIDj2^Zoq?3oW1+y1BN+6|ZbY>buf#I!%va{PbV{@<^Nv*c5eN0R;(XYi6pmPb=L znM^xMdQndq>N@5pj_}=vPVrH{w7}1~gb4x6zOiiwZpW;|>LVH;f?8U&?kQmlELRKDhIjH3vt*K#i~#}#neYFaXQ$hk*hK1cfHngaaQ zZMw#-?LMJ8Ggeq>u9n#B+%r>iEaGvGP)nqxo`hV-Jo97`abd=5OJmy=!7(x1K|s0{ z7tG=mz7;Xl08sA1i|Uq0c<~Yc6#YzqrfH;_pBm>UG!Dfz3C1-FoN7=Jt1mwbApHJ1 zHKZIfB*g!A<#^v@3=sJ6f)V^Wf4e+1+!p-t_4RdqT6(SP?dkBnhvolyrqrmuneOxF z|A_2uSP;lp~f6mOQVgLk;9(Bko-?@@Vp;^;!vCro}oTlustU4_Cfe85@Sa)a|?D;xo- zQC8ui-O}7y^mdIs8(~T7xTKyZtZc?biV7JULc7z97np8lvTXgpKup$>`{fvK9Rp74 z3vK66wn3}0ly*abz^MwUDBdmDpVXLAYo)VSO;CLXa$n|V=N`&g(5(aBl3uc7m$Kmf zbQsARi3^g}q9IWo{CGv+P(rzLh0D+nvPlxt?kMrpxw_pg@Mq?QcpKt|aw6#=VAGB`9*@l2lRd|J)Lt=HyR1 zDF=9}e1Wo55)x2y$FsynF;g~c_xEbxufB*XJpP$-T_ME{H3PzZCQ=;_bSg;w0*emDV*lk+ zl{KKNHm+&$RtdZnLe%3?RDVn~=u~0VOG7fXJQ&W&_mNv^EnR5uo59h-i=TU`=x~9z z9vSCdxmZEd$I5t=evG`^N(-*xR>`q(@bTw6?YDP=N@fRR+T@ag;1ZrknZiLK+f@6k zUxUJ4_9l~t5negLwv6h&bUWbe_;GtIh1m4~L!OhT%bJ7iP=|JuARclHm}=MR7+^jM#KVS?<`io9QOd%3@EocEbX z6+5C%R(Oq#6!t-cf(sO$WJld~M-*5?{7%lguq>6Lujpp0!AMibXj1MXX$v`R^t9@^ z4G*>lWH#Wk6n4vD)XPGY$GB_LAE0D7y36AHs}UewD8>&m$>V-tha~Itb@99Zg?ndd%ytfdn0S){?O}zSxlEnWXPUhl3NDeI zKtY`Dq0wRu))fb>@sue$V}S`-@olO#f(?4Qlv8eRTz-lLGJ{Y-U}HgE$y~#;Yz&_o z)}p9x-%OnvzW1f)HM|cRDR!RXRFc8c)Nolg45pJ0q8!&M1aUbXq{ivi#^FmbeOK#* z?r}AQhmy*oV>buPJkWNf{4^4A!GA0|SErpIv<^PRm7HL{1Zj2x?=f@4To_@BBXHTk6plGy0Ej^tD>$Ezf75bA>eum!Q9? zxd@{qp`?Mi^*hVHr4H!xHr3}UvC{tPbEg`Rws+v`6^nKE@)+?X3_A{K8Dzi+m$fT`S2QZvUO};)r#GLCPo10I( zIb^<~uNbRsZ&dI$WTW@fIS#w%0nIIN)m6Z#!T=C6UNm*2hnYr)jt9}B-!fv zK;+cSH#j8k@aTT-rwK}tKrl}czD(0y(&X+O%HNoBTt-plm>dgIhYh5K{y@@s+|g_d zS@=y|mttEq@mI4s!Xq{ScJoqY9+Dala-kKlEg(Alf$NHfXOOC^g^S8&vyh=jJN{gy zCKI?}yuorXv7+oPfi$JmwetK!H(r!V;jYjtW})o+gB7bz=u^S5Cyc;@USH)_}#89gi^#cxbvRh zCuIn}W45MBrVxCO*-r1EtVPa;Sg6PN=Ko?JtL9`KOU$=ni`5hs;uCs%N6GSv^08?s z5#>qJx?7S(dD*2Fu8wGni+Mn|haD2(QQUHE2{Q<>Y(=+kBYT55^I?-}W{!Pk%CQ_5 z#3HGuC_FkUzqY$Cp3cMwam4{5t97(od+~L;*IhTX_L^y_+O2LMf3YgYoIY<+{XD`7 zbUo9vZ+tK#GBrwHh!HSpBlDs8;l!QXb3d1oi))@b>uTq+>xS&cMmvz+cn8m;WUzzu z&)gL)WM^;Z6WiEk#TG;PTC)9$B#{(ik&xsHxe? z2bN>!WqUZKn|jpt*EjZ@ds;DM&ssla8LZK2LdK0J#QUmtM5&pvltH*i zyF-EIVNY_wVu!K`q4-PFj7 zJre$m=$j{_`<44}hgie9PjY21m$!X9-lOT)v#<@%StK_2twM9pr!MBy8wf(4Kw7fo z+yWj{YEq*}oH4`TW-IJn34FKu-)s$+orj?T3Z|}(zPcl~Jn)MjiDk+>CnSq(I^l3j z%xGP5e-Z0}ipE+%iWp?~d{~*l2>*eGlM45^RWNV6U+F)%!Uk^fLS>&e$dN0lUShSS zSLzIk>;96$1AW$f5B<%{=Kq%jtp1*L6nWV$`RmqVF7x#)80eQmq?%5A+ejvBD<;FZ z0M^8!-*hm0WCHw!0aSInrL~ue!GQvx&PxObFEC|1S24j`6#-R<;gCYtSr}QQ%7zjh zB|pl^#~%A`0;0@-b&Vg9k>$&w8~BXEmVpIkCqx@Kw;(~+(4nTJ(D@h82ii=IU3GP} zip!<}z^=LivvjR)TzXyBGPkU5Z2I2PGA~~prQw|)rQy}Lt8QG{9cq}I)iSrNidA;C^7o%)$#`}5*3JKwtc=p|=Dpni<1|gpt^Hpx=D%ap)ZG8d z$`7#_YpF|nxNl5ULbdJ!(C?!xMtpAoq*=s+nr;0s>KOH{h>;bj;}VN~FF=28iETuk ziPLk4P*7}`o(^U>j}^}Q-n;N9cHWp zr!dulIJ1!lk>(=ZDvZgfb_GKhG^UQfdJ8Jk&oE*1$1N20JbaIcj#M*F*-YYV$_*kQo;R1jS+hK#C!#QpmyRv&czA)E`m|ux>o-bpS4Y@Zhle*7Ue?M5$oO3wD3wCdF&&*h zR5O`hQ*8}VjKcOf57QRuNYpQju4LF-1SN52ss6PD-a8*IQ5+ckA|gkg5AdmRJiKFq z=M^G{Ni$NnN<`|FIQ;`GL{nuH;PlegR{)TZT{DQ%u~O69ug)2csSh`dg@ za0QYHc%$5#u=HpuyKu}(dbYCLroc9o>iIS7ye)zt*kc*s{oa<_^MPlaa(Q{4V>!#9sO`^lnJ z`tJIcTemmjugTB#yVrD)L&7|(v-~TWN*P8BthfT=651rU+ZZIFF=D*m#ql!xsWW1? z|0ng~Cdt;{goJD|dxIFNP;_dWh|HXqdwcf5e5T1nJTI9&mR?}Zb4Xj8 zg@zb*k?wE;U*?)Hm2*y3ozHGgo&J*FR!C5TYmQnMtGm1ZwXAFB6MugG6W=8LRbyol z=_T?Z&PkoI!)k8d5_Fm%B^X8>8SfY?HfF&EjM5eH7}r#YJ{Flp5t}{6nS?AgEAh(JRgSR}!0t>5gm&yH(#xW<4ya7a-uU`W32wDPv#>%?f(PE2&asvxaswKmW~S0Y3AVqfs551tQ<~Nk;aJ z9{vx`8a&*3@9qH9g{T0RkmSjPD5@W1~0p_Ha)l|a%g=OS6*3$=&qO0j*j}|(NwG}v3vWK*WROb$4K*U0!Oymb{ zr4mQkJ@@tY%68}z;;}oInIz!pr`98L8qplvNyQ^>W2!Oo_PX@Xyc|R1TG}8m^5 zuy*Ptnx$AqiZE>vnEd2qN`eeAGjRH<$)e~A*UQCD9yT?jNSWwMr6oo6*KmvwGu~%x zp4XKgS5Y7I#|*ftShdqB*&u{(RGx3gOr#D4zo=*x-)l%c1XmjWQX459bFw%1TfG}XVcnv0%z3jgBThHg#>s^ z*WVj6R*D)B%`x6*oQ&R}x@Tyj*K!Tzj6csFCfI{kfSRAlUfM~6psq7&@5als?Gj1& zpGkpk_NQETQ-eG!WwV|t0jGf>_s18(^hdU@7)BF^ATtt2dzai7{%4~RNR5`qI(z1q z3Y?y$AT-PLpy~6jzx|!SQUzYX-;U2Ht8W&O*y&PD`oMt|Se{;)l&-N>Ww%23eqmC5 zFd?H-A|=Bn#ej_hN+%}$LBX*N(IpH$fRXpRTujiJ?5ImAw zOcKRf;G`Hdhd&Yu^Q}=Yd&VX2sWi&)YN>qMjRMwKn$)h91@bELh6l};yQI$c7As1zPwk8{+*2i~;6Z)YD zJ-1piI3|kGpEa@hcL)_Tp*C*>CzZr=r@8RhUJEZnW5v?am+g7JTYG&+?qQg<;>CW| z=#d~QL|dX+B0tHn^_{2vK2luN<=p2#n(97yheyV3weSAtgDvLcPbRYV%Jv0YZ{T-A<5z$owuFC3?#Lo_ zgv7DJ9Mi@)Nen6#v4yoD{Y`aoAilPBun)tYlr3hGj71UMS?$$>m5M1|&A@r;uc;?s=c3${vy;tMvGAZ51=b+^W}vcO4I6>2h!yq!AnXE+PO3si8;Qld2vwqImh z8RSDx1@eC3>2O>bb@isTwcMXvM@u@lF8Q*Z=DvNsoy3_INdqu5@ind@Z}c=w1MP;Y zoIA{V91>EJw;V$b-wJ2rITm7*yun5zN$x$v>jXHc#{j|G+tW|sJi(M*To>e z$^dpBX?EQE?bizj@M8zMy&&xPK!FXmUKV{j@yGtfG0_-Zx0|+^wxOAoOX@zZSE4yp z_~{h9Pep-XIOSOaOC`n0nIhN(mqO$j3j4=6L2JH^8dZ1gG1+V6{6(%RRy^M~+7Isn ze{M#eyQ|QvUumeD3|g7?81ZG8uH5{9d(=HN52T#&7@?*g4mTI2wK&rs7MlpR@>7w_dE&Thtsy~7 z4g*|1QomObzQU%sh4?M@C+hkrqzO9DRXlfO0=1%PBWxJ}PljyVtwks7gGy^_GwteJ z8$(q=W@4&jLt`YO0!dV(?jD#jeT;TPqFS_XOc-O(saeBDLXeA2=_s(n1!NtLuqyuFIWWlue#qqW=^nhpEa&3Da1Tu~Z#jzTnac>yxG& zA(4W0rYLIOgrp5=)MzwF&0~p+TDa5l)EbmJ5eBhmHVec(oJGpfsU=ZQt#`>1=c?4q z`<^7&UXY3^ub|`Vdj$z-;Hc$s14dK69O)85tC)-Ly{@VS!Vjwt$U)sRR%=|)Y!G0D=RJ8-=la%rH_ zNSGcZh_1XlT*m0NuFwh3J8Q74%Uf%zUU5ww(xTw&jcd8NYWr)wQcZT9Nonn z)0oZx7CH`gYJb&YCF(qc=KqSeK%H3>EV%mPx_4{G(0-F7uM1v0RY3w`c~+oc9%XRm zAgWfX+ipd0o5Z3!+p9fFrUe9j_4G31j0iO014 zCdH6Kf@Nu>n2D>n=pI{F`m8;1pLA5*sNlGs7Ni10DL;?#&RfdSc#69lhh`Ag5?A&UW?wn*{J6E-wivg_B7BPXDUj#cYeo?+DDa_lw%Sx>i1@Q2F z#oro!9SR~M26V&%4p%<=kL4d?AdpoR0{{OQ;+3+N-2Wehco!Xd!+r?MdxY4XE9qqh zWipj+ViIlpO~L`lp|{YYxKT!ShN!Q0RW3_C7;#mRFa5?FcRb~!wux5`SP1O;ZKk#^ z!B*M3*2bI%_WuqbcKY7s%aDxJfCzg~L2H5~!k5c5A0jhOGv~%4NfnbkNnavkG>Pl* z^A^|8vsKsw)EYc~40A0`-h2}Ez&HCwlE$Vs6z{mr=1iqs zL`=15J2BB@ker8~>OsT=%g$lfAg4)0b5u67@hiO3u=%$knYNy5A zdBVGpd#kpgSLmHRvn51#)e)WJ|G*GliGG}EGZ98vxba)IvS|F@7c+I5s^a|eyP^EJ zK7pT=@58yVF-b{5zu$|^>?zE?`9s41laY(rugkUK{PAc*l1MzlfSz2ksw)19lscMS=8Mf^JA@v#sCH&IVm3Vf;uOujA{fz$M$;$E@Ew zD5f>{2rw5rHUhBC+bz%e2$2Z9%|G0pXO<=;cLcns^u{*%#vMzDXKLTx?^y~C(tTqc z*L1FMs%ZkbFz!Yr4peky$G0M<+F?|zaHvqcW06D>k{-s0Huxz&wv60ZTZGSKD<1LTXvjvWsn5 zk84uF@I8R{@f+Xsx*xYW`T*Vs6X!iEb5TpPDgD|_(C5D-X~%12H`co9oRk~;>83F*0C__Hg(0Npe_rSMvU9L`lwZ}tAUjmX#20grr(*6koDY}VSI8W<#uc?h|Ow= zb8k!r0~}YJgra1RGp5_joY+Gb6+AupGHcZvQoqT2%LYULbAfK65U?SerkBtLvJP(n zGt0#6xHxkeaYMqUYCMWI=x@@sI_NmWp^a`WO1*(qyw}PET~vN3ZMmK8eM{{KysBWdQhAprs<0*u z$9VV5gJq-?dWJ3KvbAJrHTT_WjOMJEOi$l1_mD+Y+XZx1_0uRO}bp+wVH_D9xB=zsNtV>BHvOAa@!Bh;D^hMOkR?gWSz0iUtJj*QE zg=KlZf@R1GfV(64pgC+`FX1%<|Hhy-86KH#MWnL4mCkeY(ge~sW*kDc;*Un07uV5> zq(OE>23e)Ec;29lNS;Im33#8mB1a6yGeqvbNYuMxb<#i6O{hoLC{hv=#D}`b)w>K& zKp(3x+UnNLB@za)fj(67>up`Hg5pTDWOJ^2bDsu&fI*cPhGtd>Lv&Qv~ZjBLZ=ORN7l=ulw{L0IRx|ow{m3ngQ z*wrC2)%4icxbv6}<(93aSSu4w@X=|vEC+=;Y+Mh+29&n4!q5I}Ylj+>Ti$1T);`n7 zI3hZh(87HXUNpLh#hx-R(+RR)GVI1aqum&V@C{NbuYvCMZyV@ye3iQxj4#Tx%kDVL;68+nn;Me|?AYFZC8VdT+!@3!QWG@BT~f)PK_xP&^Yq zKByVNG+3~EYlkcNYTshJ=2^N=I5uC>^U5^?{aB#|t%>pVTzx`$nlgCk&a`gk2SC&V$aJxmBmgYhtdte?WAw~WZMpILW7=wVqPP?_zELN0(dITvNgT-52Ahs`+}2?{NTB3Dl=i-c4-YYm{}~EaZAtW zht$d~yk9*G5?QsE^8Sfr+K#G(2Q|ZdNnC@0Hk|l+YQ`s+Sy)4#!J^h4fh+|5PU9$= zJMw_hFg5hFl zBO-_`EH9t}P{H*w)t~lO$kLRNzPN1?l{^G3o z!r*tNB(gaELi4IPYR3ehUsGpIermlF1eA9!Mp3Gk2U}3eexay1-n}4Hx893k^l|AU z3xYLl@SF-#lb^GaU|IeUe$_L8KYp<2S z@m6f+jgQ59dr5+A>#N*@awRpzXDzpYK#`M+-!57{2}P*f^SQ;Fn9Y8OD&tH{;&57Q z^w)Qpi8X2T9>L{R-J0H=UDL{xy0CD&?x}+V?G6YljO?J%*s_@CEKqOaL(nlrlar`J zA&O42d5o|heO16{gnx%4z3ckToCxdjIkurcGX6>iS5xhzE5XNeR~04!V(DO`5`(=t zFsaZzg&&q6n5aDRqqliMRi|a4wrCYHDgZ`uL0o{0?>$I4MCgJnQ97#Ne7FRdyMsfW zy_)1oI=gjGB8Wy0qq$}Pg7ibvh=!4auFK2vOVe_aHnZ#GxlEgZrNgfX6Z^+Od;B<= zUY)(FdD+Nwm2XL{U5l~x-RleJorsCHo!;!2JTKU9i0m!20cq1bmBh2hfSAANX5Ev)prr7>H`}|fBt3}&2eemKQIOW>jg?hwKCr5Z> zr;tI2-#6vk>f@Se^CaRp1${>&9Lj&&>m80fNq`m@4u2pNz+rJrnqyiqA-mi*6C{cZ2}$xtZZ24{589Kyv25r{Z(*3Ze)W(1X(|i0zrS#K zNXrT_8Ri61zo^;y~pA9i>zOc{MML7v2g?=&Tn{bxk?Jya)b(cws%k(H6 z&#hqydU!a?i<|}$YEGJhW(?t2GG5wm=twr$*inG!{mCgRSod;v9c@-aRj>ka7yE`$ z{tJ^ctUeP)B>F*ixI`4c!^!X>QOA{i{X=Pxl|I z2OeQVqTQ`p9!$J{!G*JvWtLqy8`8y?+4Qp}jP^1{tbawuzLy`bC1B#=+v z8F+M43J~Y?V^KlgVb>%)s7X>#=GI_!9?Ybj8*JsXIXs|Fq{|cQVix75M0LzGFq{iSk z96};Xh;@j7ClI-O?4;nq_W&<(YVXQW+wx-ax1I!5%~TW;5X0V%pD|9%0I%nnw5kZu!E2kgQU>=|#@*h|u!8Bvpd8B>>A$=C@JQ>tGRrALx(ToiL*b6JPU0}N))3*1 z4R|`~TN6v*Uz|4v#a;$h$V=vCt&e}=3D88|8}A@Q>TyWeCC+=(IB>gqt_|JHy9a*~cBsIs zD(3NjA7TbTjSs}DfQ^Eu27YcuW0bq=Ir%?Jz%ap%jC9_u9krX2w;;)FRzr^v=qNa> zU^Lm4XaeH&Mz@SD$g>_d&hg4H^Z*h#)8ncSln;qt&?y-K@Uryhg~0zl2r%68m;`n& z^=tD<(lw2g_ogFrQT98Cfakh6tyNvgC&E8^-a}HC$>*aj}`M$tn+5$)(p9@u}jg{C4ymT_qLzRI*c^ za(0=6co_clT~+c#C#lmJNJGzQ%jx~Ml-=Qmf&iL8WxxAz75A{Q@xKLxBrfB|4*j> z`4ZJ3&gooq?f<*>e{F$2AmutEXupyA6G4_;B~qClWn}3r@kuCoZr&nn56=Y!TOJ0i z-+OwxMAK{(WakRD7L~TPx_1^j348wtPj9`IW!sKAWUBlnydQ<88u#Sr$6LU$N5Sjz zlB;bk%%)l!H^f8E%wXZD4EI-9=ifcva_kz@^rsrv_cM3Oc*tJzH%l{y02?km51og} z-lUfv5_(^mBu`)9D7`iKQ%(?{T!FqPnL74f9HSJDYvtlgJqeqQ|Fpxk_oNE32Gl1Z zx&yj(DMq5|V(~t~XeVd%4NnZ%TR*|N%)yV133b=5e?e}-!4g;R5eLL;tynLg5M;sH zbunI;$@qru$m!vOv6zg9p*!z#k;tIvC*_skf)Kj5pgWH)8sd-_X_LLxgi&b2#| z-dplAX19?w?Ki}7>K@njaC3nS28DA@uKQeB_vk%y~;Kn z%ktg8=pn1u6H*S;j87&~)@FisH1=Im91|rS1gKka$qDVI_QU550O^_>94~pG7oVvy z)lVo2^h~Pxs$uqgqtI-V;B2EnS}Xk~89vJ2&#R$Zftk5qU-y@xrT4z?Zx2gCjJy4| z(*S{Av){Mv_jdpQF~H~h`Ljwh;Pf=#A8Fp|yIK_}%7A*r6arrai@2b`Lqd9-M@*4nD7DC8P@HNuj{@zG9O zNZD+i)CCs@nr3J4nMjI7v?0rH8f!|PRB*Q@%VZ#ew(xu8h~I?F&Yaz8Ue3;wCh-tD zP4DO{?G&E=%S=2wqV28}K))S$XAQQjoaEi&H1K^jmdPGT3OuBvOS(EC?j-3!li=`@ zTHOz_Nh)V%9(&fiyse%;hQ3fZa4~w}Z9Z5PHz+8x{L%~`J>P^V@xD=sv)xgA4Fm?M zE(o|5#F-*SSNUyE@-}*wUAG=-{P5E9#V-n-z0HE9%idWfNCbfymz<1SDk4JHrNk&em;P+rCbRL2_EaQak2>;~SnH|AB|&=`F=Nsn1#eY>wVNd%mwO;Vq5Qu9FhI}0iq$wJz*Rvt?!A1JwspawxAC71 zzoi8ewMXd+96dg<`<5-Lsk&)TbT%(#-X7pj?9jkh>|=c|QyCmzR~5hI1zWRMDx(ER z5b3U>ixCPiE0rviKGd|(6c*I2I~MFpYsv*emz0*ULZ7de-%48P8Lv*uaF>Ce9oV70Hu|bLX6SW9VC@}ffq3&<|=j7Qr ztZk}@qaQX(esK<4{>8=BKd%4d`X3iBUthikk-dMCv^!bS19RUsqKn|x(jZ%z zvjw?I3c~6>=?mBuBl>Ysb5L14c_R`s1CRq9BqE=At!SY}xJ-&=!W?&pa+x%qug}ic zvJvS~x$X&)Ga+juIqJ6oN>e2>f>x-=*x7W74sTW%4`o?CS9U{}xTB+oXQvKnUj ztzAmLxneYr`qD~TZ%_x@Qgy>}{K6=B&J74kzHQ?B*!Hs?KM;E`UL5?0TCWY9Nh!k7 z{Z>oV&rkEWqS(EYvi{29X(X?l?@=P{xnV&}qCk5NwjI5x!o$}>dAJM&Jyhe{?PnXn z*mkQLaiPAhWD`7r_5`f2FJJ0)q9ssJh(ulO06Aw5A~WV&MM3#ny+e9?qK%Uyuj!7^ zLdg`F)VO*3s+iIL^v_FKZ;};1V(ef0G-qBkW*Z#C#U#=-_*Bb^Wh`eRi^_m*NrZ*L z>C5F|0}f$Xy^%t1%;bf2X(LTt@qceqf|J~6JRzV*0yb4R2O9;pm=&l~I9q>fmaNbt z$cw@jR%g+`X1=9sMriF;Xe|j7s;SIG`8Fog#BVQO21!08XS$8&u(qHcdo9sE)7SM@ zmV6Xd5H<^u6*8U~5t67T+xP|KtzpolRmFm>diySMZv{#AtvwU=R&3k%N$>5ghvAmr zDppNmmft=wv-Cwey~exT2N&y>)ikGdc&8wPb%p*XOm|cj9bvPf=?>D2v)K_=8d1Yz zZ1$IDrHXySMe`ApQ>K|gu3y2UB(oy~GmDX^xwU2_2IE|I*2-)|oCW?uG0cob_;f`@ zF1LG))PgYUfMhQ^j3{O49R+~LGXfRtmPKGc`$*yX3Ec`i)j)z2bjb=66N$mmaTCF$ zB|9a0Ocv9!@ZZUTVLpoOlqT_6^$BjA(kHDZcKZZM^yk#4CUW-f^+_Mr-9EvI`T6uI z$w;MGmdR~-RY|RCPC5h3qpbrCaU&)M9|P~~5~7utghp#*sx}kt|45W+fD`l&6Z?ja z&rMS`KZKi_$UgflGHk|qK=vs!S^(~Nd@@OYLi-Q5$$98GgYQxz)Co$=6igS^U?^nH zJl844{l3Q}FMVH>qRlJRrM^b=3Y81;9l+OnOyW8DAcz}0GEbQD-b#+G)F?$<$?s4` zwFbI**diMxY8%<#Fy9JThPvHzt%7&&i<0%m@@K_UjJRl$jw!l0%-L)0yNh^G6qABcq(r5@rqTyGV9 zQR!~foEFz*7@nN$!Rx-hzUV#<+K$O{VemVl`Iajc3%k(Q9b0asydh`7(l~R)*XoH` z>~r0NI50LBg$rr|O;^+|H3u4*xn96TU=4{lx7axf-C@AG5qhd0?4;SBs(5Z~bh_Rc z1+|B>8}0bun4xU@aNwdSXk|=I0~h^k`y9y`#)vYAGopJjYANS|wR)3Nfb21(PkjT) zMN#OHqY&5|*JV6kb6a`D0a#+Bt(LI7o{LiG z_lLLr`Fuexg-Z{)C zHj0=bUa~cZK*v$i?!-9tQUhiv4f2|7`FexGCZT}1M6C0(kaT<$UJ+|g>QPdu7! z!23!C$(sslKGclnySb`qt%Y)i*-GQ<9YR~OW0jr5T0@Ric8*ncq9b~&vNLwqvC7V| z%Fh0M$0|GCb*!=j*RjgZ&~>b`Gv2~um7QaioxST=W#?FBCv+XF>>R7?nCn<&=U8QD z{GMZ#oe^gva@#`tLy}>U!Qn$tg>R7?9INac ztLz-B>)bxpI)IIH*`0_1Uk+elIKjjAg~w>Z!h+ft6N zAT+CFAjEas6g4m1%ZXwTq>g8#8#stWX~Rlq_)BQYO!uVdUvw&JB*AoP?(jf zNmg(cu+L))5lau=@6eBh>9I0h7CA`bWvxgMexPjtzMAA*8SO~T2(2yigNe7}4pToK zcY+5MwgzwYU*n-CwgGx>vyE@9g09_0m~|q!TiMIh#GtwIw&EyBSSW(s>b1=>ro{r< z-I4Bv2X8#@jC{?MqHA^)u(XdUuV8F%Cj*vC(J6HT zoUXtObJ5SjJ?+MLk=K@ZOH`i`A`ehS)6KHtY(-Wa!m}XYX@>AA6wMDxu?pI6)C${= z0MjrXtgA+3v~HzHNMGYCvW7q^BVhmj>A(FeS?=`dI!aLsjkT<4;Rdl_Vy)*0SPkGu zcc7!O0Z{rQJOIzWxLI~ZhbK9^ed5jAypZd~K<%f4!GZ^tXgXHB#R?)Dd53s!11l_^ zAy~??DbQ;Cm0sB}VvS<^c{+ppDKw&%bdUR(SUTH8X=2fhwNUf0aaLsY*{^6(ENOQ0 zUVbCj>aBQHRkG@~y`Y+_ZyIqEg3523RmipR+S*X%p#jC5V{?cE`Ea+>>?3d0Q9+n) zFRDQPd&iluS-^T&zj*rlKjJnavLb(c3h}6Fh71XKsy;Bcv?#Qe6*aQf5+;6VVIRD% z>$D}DEK70huq}{ zt7?gaLr3{QzuJfFu$S-|1G6V#P?0u+!}Sr*7(GU?Lm!}gV@gmzVACSZ>VS}z7KfkA zf~Rci@7g|f_q6`TZECzLFx3ygm7xI>cQWi@)0FNB+Y!=lW4$3@_p`VKM(SP|+zLGh zhA=sOe)=H$89Q6alCGn51$uq%KkBE{YT*xl2lIEN!=<6Y@)iI9ASuu>X9v<&^@-Lh zAmonj2(4rz@&#H5oVn(8uhDO$V6K*8glf0iCKzJ5AkP-hp8gW76zO7s#=n3>>F7&HDThGS5j=5s0oC@E(_+@r~Q|+?=Iefk2se1 zg{iHk|BbSRxBOmsFG~zCAtO_4`=I9sc%to-ZpF5#p_W#JGW(bnjpDZt!5>m&D0ia% zf+Cd3X?)2?SM_iq{ZqA{4mnUZeZ1*ICi7F}GERKws8N`E8;65iB~qw%8GtP<1$V3w3)Zp8Zy6;i!xiecIE z!p&vH%7SOqJpW@2&;6s)5B|ga_G1gog`OV@rC7*doTa4%=9$*_?E-A_Q-#_TB*wOD zDX*l~`35MO)j|ta)dS|tz`PZpr~zEq{?K>Fw!PC;re>B^NoNYs!Uc@d_>nVs}HQeIeIhp*J(q6S+2SPjC-<>!6J)wEGZhneCBdq8X@q zFefMb$zpTetkrgZzs$*hs)6XKz+x}t`kP@e*uCx{;_J`RyPPeX{arZ+H}}BTceL8? z^BgUdQ^sP;#XCmx@c`+;pi{Bu`Cm!v_oqa#yJAN&%(fmeMR=9&^y^^+Wm6;C;0o%b zwLCkksMFNPul(8u>|@x|}W$n6EIzz)8yY0Xwman0)2-N_oD*@;rw`#l_^ z5Q32H$=2Pl8V3x+=GLf%2pXbuirE-PLS;0=ULHaBcU zE8R=~x?#IEhdHa5N#xWM4$IlvSeE19=uoY}K7GSRz!4(i)VPx03qA58X^m1+PsUY| zx2(G5?Cyz?VCR|%&2d826UgR!@*DVPq|5K$zI+SS;tZBgqx42KwKW%nmi#+bDK5nf za7t&S;rXA3{f?1wk=C^xADe3sx_7p0CGrrQ72uu3(nA5=1LklU^;Hd!l zbm09%Lfu$1;~-ms6;x8)qR0S2%V|f>$yr&*9hd;PGE@M%iW=+}_EGPop>s&;rwcY* zyv1feLW*p$18X+!x@m%HDf8C`dM(nqldJYtpL?T9h2cM1A*9%^F-wfW4!7o5<1g@s ztdx4^e-OY`+alm#`5&?MRtyqAF<`Yti!x!oITO#xZ^?%fT9%70H`A=?!w{?j^lE0q zuRffhrZorU;={>egd7&#&*#fs64A1fTg_n_HAg5c8KAB85h}8j=Jqi){L!^!@HO3T zJs@G-_N*>?eK2ZlTgrIwo$WFjpMd@6!socRG=~p>Q~`{nn5}yCk-cq!qHbf*pFs1` zX9=p^TUr8V9F5m8LH!j|vZ*oy?YpHr3qZNtwI+sI%ECnL=`(#ix};)t#n~+*v|94I zqSel!F+!lm4)(0xQvYmNwF4_A`F*k*vg!*I*h6>Yz)g9jbhb&-SC_} z@g^}O0*szIW7Z$?xby#-mZ0as{d&5QTf&^10KzWP-^goawWfmq#0;0K*;-cm&oh{m? zjI5b3kci-96NHwUuT`AZhV#;$Y`@niBWmOM@=bE$VIk{uYu;hdI zU3y4if4anA(4<99)>_VhY%hoC6>z}RlKgnbmOoAtKaKkOqM%BR^~Z0rxu0bE0Ya7H zYtf2ZS8~Ag5q!NBRP6KW{75 z8Zu}vVnE0ScH~eu#BFoK{D*eT_p-WC#5$3Y3d`TvC&7~6!FMLF=?x=L1ahS4p>Cuj zfYIsZ8;gI}oZLUSN!@gGX&O`?6JEiqc2ev-^0gS1dhRw_d28pkH=|f!1-uPd8Cgp++2B0K zf^+N?CAsf>^T}Ahks}=(P1;#R$&JFXdCZoD9d@?W!5^WKqvp-YZaol8lT4AuD%!I< z3}C!~FA3;6&1P^>c^O;A5QHhM|CfIBmB;~L-3IkoHQuau%}C0TGeXn&5#GPR#{NiN zd~-=?wQkVI|5o!WXL7e5JZSdq)5WuAi>HHXP);^{y&3W|9u1*KL+F1uL+C#{yEMW( zdRrR1tE}Hylj5u*tljgK9EvqI@mkVw7kgj)YPn45J5qg48_3 zPK_QI8;>_0*qG;fckEJk2JN2@=|^))R-|?BoHAx73OBehotP!Jg)ARP&4qN(fkC_i zd!&iSv8VuG!!??ldICu*ZPL|Hor*Ab%?irL;+nfwScqU(AS$=uxkh_;YH3?NZA|Si zS+=kbAjz4jiCHl(&7xB#5^b`hxjOq^+(GyNX!OgSpj)2VIl3rp5xx=HUFZPQjy6)3 zZ?{d2uIfuPrhqgbEE!@BeOonb7+W_OZa%o&U@-kCKPgP96#}$dWlDVtG<Os1#jAPPPk|Obwf(v7z zg~6b~XvknJV6byg=?x91)eYN?On0C)`=s#YTRMc)g6!)4cF-V3R2nu^5++nh>zY;K zIr-mbAAa{~e*WjPvmc($|LyO;JNvMJ55GPC^Z6(H$M4S1&(41M+v~62U%mP}KmYWD zXtp=_*Qc`|*sH%M5;;Hr^S_KV=pf9duGIMr9vN+3irzb0`VoSID5lfM6%?syYR|Md zHWgTLYqH$hQG|^DYVKavI%?FDgG{ydggmabUvjM(Ed0IIWRb`I<#larq#m0d3tr}R2RGg93CKdG;4p8?nD}`5v9yj_YDt7|%L3#(%we}O5x89z zZL5vDn0?F`%S{u;j+M`8a|M2^Sk1(6u9dTTfARZ$yJc6Ehl;q?MB;mT)rJDx38aZZ zIKWC)AxIgn$mtiq-<)DtR!{z@*t#ibrKh?=7!>lO*(_W2NqyRW+I-4C-F@1f|2RPU zY|D#++Yn0o@0N?Eh9t43QY5E4MeZ28QDnzxr3SKCyj}nFlijT6yLRKA-+7;N@AJm{ zy!AdS|9f@*=kxvj9pFZlY-)B@u@(OqQFGqwA=G!15^nFzRK?a@L18dJ$=*=ckiA+D`+c;lQLE>`aNMw+!#03%(O zLF5K372StZW$Nxrp*FmXfeKRaZpoD$eODG9$>aLF6f^Qh*7~1UAGwNTs_D6WDVcgB zYq=9vTy=VGJBvpY!55Y&S3C)r;K_b44x%k(C_-mwBTMc^^O$5Ld!^beAUF z(xZ>Vs^8&{od2rukawDaO)%#J`~ngDP=G17r5&jr5n8kLo|UzGZCSO3Fwqd|T2lR% zNo7z&1u5cofN@qccfro6ag?awI!aWd*HNM(M~P}``;HRTQKEVHICCSD+ie%;nblgnQiiGY>XAAGi7SB1_Zu-) z1Y4=EE7_DO!3q~>wWkG9O=+$11zGFoO}GJ`-LN`GD^+;7wh+5t%6vht<U==m|0 zOX%$1nk6gPNR~TL+(3~RSipChus(*R@jFKcZPe1Uf;uV;R6!H!&Aiddqo1H!#n@we=3(>CMvFiyl` z40t;ngiDN^45C`jD=T-q6O56Qx8j|Y^=q!+*gS#zPc8s==VVM2+e3D8Pqbk&fi*uu zb!;d2YyO~c(W(&_$sAvE7R~r-sh`6TbWvA3h=|<7n{}muM*8%vV#?Kkum_kc?gF`O z0DNrv&PD%xUFgvcFjsCMxckSlp2q!y(i8Q^Xxcd)Xv<#yxk)6Vbv&Hu5Ekf9{3G!? zSQ>96?16;EVS(FKtHZZT0Td&>W~dg6wU$=N(Z${v>*5X6sL0Cs>YBeb?F zY3vk}gb9U6T|G!B)2weVUy`Tf3{di5iL59uFvFIPmRRdSr>!jEg*#TYMrle+2#Gw( zw0}Gvat{w6c{coN8eNh;gJW|QIK~oZaNwt;Nc>E(Z@6eaViA-S)3RPt*RS9q zEE(~!HfjUvHM!Ai{6OPzslbn27%7BgDmuCP02LMaV3L8p`Ym%Ahu-`g57GM_qH&XV zF^rREuP$C57y-C)Q-@XoG=BH=<&b$Ur947ffV3P!1sZJoJBbBCsB64__!n$D+>+Mz z_q(#!cKz@#*!E`hNZV_>KK;6FhvVp%YJ0>st5}xFZFyD6)ga*1UiZ*e0uPzYHGAal z6#`Lc85uRFfZ6CkE-)&G-H$iOj20WI>fm_njYswiu%*Gh222Wy zQ?;;?sEWSFB9w5S@m>UIYR=q(dir#02{6ZPNplLFVj>Uw8cJGkX2xHJpnu?~L~;LA z`52;hotWkZh}v|cwl#2)t#IFSMNYo4V|-%ndGq#txa&5lAMiIZU2<0j0I(mHg`Hr_ z-9kVg{S6nn!5B>(83!JW0!-^{#H~jw_MChW#0?Lbw`(rw{j(WqDQoT>iaW6)l@QWD zC?{ECfnB)07X&DND_}wCw$8Oawok#1%&-NKBxUvoyQ7mou1c<4Yq0m0=K`W=Fohd>OJWTRY(yU$w&FIl7e4m9S;o zB^nP^Joj+`%-mtRG&~>I1johTj^r++=RkC0S}_B}4Su-Y6md1w$Fbf7`X1^dqLb8` z2cR{T-?ABDw@eT|*7@3p2;;&4PMD(@*4EFe1_r&{DOI22sYGG-vf8Pv9&;^sy@7Zq zcmchTBo~&RTUriyC@`%c-2l2TcyJnF%mcHAp@ zb{;eV+%v$#EU9UBobT}=d%|SSq5d$o94kQWR}(u`i3T^nkjElo2pqG%1QpbF!(W(E974%G7$R=!=R0o*XT%OO_>y&R@U2zUV#(e97dnGx(j*-1s9yvNKz5 zq`V<#HaT$3>rJy@Yx%CHYTtM3eSu|FzkGPqaAjKZzOq{QC*wSB@bUB_3Q zTH-+*Rf3|BbBvv8n#2%2^)0yk$Xalx+D&-7=q^XS30aN=PyHeh$s6d+>ZgE?f0trbpF(gIB9xzz=U&eJ86> z=~fsG-xhB1{vxX;T=31gs3vR!a8;Q{G#|ES)1d1r(4%ox=4CHDtn0!G8;!!wts##> zO3+z*9d@!sVucSm-5$vb8u-xwh+brwY{V#VX(ynt|G*Hl?A|rdVq;PD)|WxG!dzH@ zKpN@C;3yb;|5)Qq6#&bjm4O@E$U9dd0F%u}4L4!2Am@rM3oHS99V)um4aV}pI8Cua z(m)k-D}q&QD{q}StEtBjF}0LjFsfKE_GX-cK>PfGB!0RS&Sw((yVOBrFru^}?t+D)`ZOekzYyjpKuZd6!W;r}DC5NB<#ztig zk8507+_Ibx)s`9_TfEy(I{u2X`j`67r zLdB$l#U#ST^cRY2o@K+etZCsvtbp&vE||asSnb-FCb=SZf4jg~aEu5owPOztDh;NiPu^&Mk)yj&o{tVmXv%@Gl z5Tz($=5fbM(&=Y!zW?#(YSLi^Zj<#Q$ju+25ZyCaiC250OrP~z82u!?8B-i=weJ1B_$s{Pm+Tf&JL~77w-nQD}p!v;U*Tyu{&CKbqVxP+-Ah5f67I*7{j70OhIeL^=Id zOX~u%W2`Mv$QlU;Xd+CR9;Y)1j{8WrWx-~omXZ`wtj_^!^gz*oaUv+cEJfbI{NXtf}3WVNM5VK8uw*d8`~Eg2CkV@lC#H-iH3 z+!bdsXx>-|i3%v84j>4o5L8$gr({{l8zx9rG^+NHc-xH3iqU+I`8Z%tGZ}6nG05_! zEzM!Y{H-mwYRiOt_kz$>&7fke9`Yp{b?Xr{x?&NS4+8M@wEQ|lvzoE|3tj}`CQ>|b z#2KE#natgyyKS!NSiL&fs}XKVY>nI!ElA!lbt?pMaCw_3AcSy_kcIdFV>#7LYYh$m ziGXJ&CA?*oDV6nu_3Vieu_ufpXdK2r)|O6yG0=jKs+p+2YtQy@6Y;=eqUM!zZ4dyQ zhT&}lF5&J#M<`;v1gqO*NplWq)@CFPqc}NlM|5gT*6Cmu&BQ5bY8L1dGu-X8MGh~{ zG&gxI^Z2cOY1n(TO|r%`U~ccMtzCaSV0rI#<*rM&&h8)B8m}H1ufM^ z3m%ALc&5x33o{9en?J$~b#D*3BJK6tJ&EXSaBrNaAb8;Apl$(!b;^TacL@}&bcOk?#Rv`BsYddtnVwJqLW@=(P=|7z*HV=D~4>wL9GJ%Ofszu!tYN;{+%B2Y4DpzodZ zT8ks&p?z{if+(T=tKMr-o9FJ2qIb`Tz2Cpm4*~+VJ03{F?O)vh&#Oxepfn~)j3=}w zcM=AFdtuJF6R~l4{Wo-Tn-i&ach29qg$Z7=Hpa1H`aRhCm}FtEue$Nr8(!t}l0pR3 zu4HQFw>*)g!@f?sf70ngAYS``8HNF~24~}waFtGA#*#Z{=k+mq7mJ>2ul+c>o>%j; z>s>4wuD#|k%L%sy!sq&pj0*aAO#!y)UmgB;xXh!OXSZOy5l}{|CG{^?J*kvsQ`j7D zhlNsaprJDivT4!T&=O_fX2Nm0B@W4gdmX4M#^Dp=MmT{3Y0jNEd?Ke2sMwE+$ zSC~S6R(kMF0-g)k3NAdBY~x1R!F#AC17Lh(YO~y>%diudpMlVi!+p^e8ESYp4LfmKKA0GFZp@hC1>8)>`8!Fq+MNYwa+oC02Z$IlgL@@w_6pI1~J16n%+Ls;&hNt~ANm#`R z8ScF&1tmZh;`JVVr4YJWF-G`St4mbWB9 z3I;emr>-7E%2qi03#;il`Bve;x^uyKT1T|)1k%1fZfjAa!o4787e%@8-s@?eDU_j! za@8A;PMcp;603j%(T8pRQExr!tw+7Jd;M3|TRq9r0DB~sOVIA=BGg=){d_rs@bBFXT%Hq{7D za5%lQ?Z2hfCr~fBe|2p(ez`vv_UuP>#%fY_8a(1ag!+L{zYC}P)yI+wt;>H!c`+VU z?_YLVv`!DmW>FoV`Xs;FaFJ+N9vMUZ>J1fIGQZ`rNf`kX1KZRnAz+gIl9_V+NG_u# z&3L^t^)XTvbdW-7S~sc{A3^p2C;S6CY2A!Jw3HaCNZY@rptJ8i8x9TcUa5iN!%roj zzvT0b=cGD$mNMaHHCh?0e|1)`E5w9_wE%eVpAD;a zm?mh1hZG@E>ZPT4&MH`G#$Oee+ouT|X*I-xT~4SembkKpD~}D<7)zuWD2dX)l7r~T z#7bp}BuT%bS*UVZI?YIhvxG`!{cG9+!z)thtbe6fKAS+4>{UsNR6^@t2b|w&mD2iG zYUQ*^o$pmy8>yr=iDabe+N8>A{p+B?j#pmmU&)o&CJjutYTIaKwn;SWconyCP`+`J z302)5rkQ(H-X@U^SA38uDJx;YK6P!+lTdn2XF`qxCoZEAm` zRp|QHeRRFaGdQ(sU9xW1scjdjVwXrXQtNK8s$C-KwAy#B%3ZROl3WWfk>b>nwEgR` zG>ZF$92!gHi-MwCV5$YX|NH2FXcKFuUz3smGhs_3?F&~@@ZbR* zCFJEfrifDrKM4c)-4IDHW1lNC#tCG*xHS+daeYbhir+%Aber%YE0a1p&2!=Mt-b1u zdr-Vp#Z)O%11$%;ZzP5dAimkKt(ufgRZ5lCW-9h}4;X7qu8~cx!|cwqeE`z{33@io z^P&qUV_Y09c1DWxS&poL9uBz)k4tVQrr(~PVqAyk1=8w-4@;T0T07l@gkc0+fjVSe zxX(mYbu?)b!9}RYne|y?Aiw%sB{PO zD|zSeQK`WpGk?Vkrgo}^nB^3bTV#qD-ra*;M+OMcx(`2)2S^R=gI4}BcHK5GCC$D+ z#4;iD-L20=Ng=t{Ns?|{X`RDtjQ*U6c0GDSGMY|jT3(t(Z4nGC2xAW~>8g@jR&Us7 zjDrWLWbD?AD$Z0*?&!{f(@|H56^y(V33`NNH|)Pn@4#3=OR;BwriB!1#q%+u4tB#K zk@S|*!Bu={$-hI~r&Kt>i>piXgyStBX5x1&Co0>pEkzwA$Pt`@6yX@;3asBIDd?J2 znxu`m$)vbtRSm%2L%EK^cvj1kd+Oh9%cb+(cj?mG$M(%H5Cq=acf@N zki1N8w~gTS?g_{q`goC*dXlqSRyiT_fQjIVGOwB|Ywx;lQV)f8-vBD7H&@ypoOibih?-Gw8 zuJsDlEFluZh8~bwF35|RdsOoYgx>)m<`;C!iWgK3gF-yyv_b^*Tu;mWCN_>E)0x~? z2Gi@c#XWWg8;68kv4Y(qu2Ep%sBpC2nTZ}hveFz+9*|A=a6ZI_n0#9Hc$^2WgWiS4 zfE_`tt{b)smVmI=mX@H&I+Q1WcTfkgWRx3=8WHP~l!d~|z0x2{R;;c#yJdt{OI}wL zfH84i;Z%#6bZr3~aC`&Arl#6fjj}mBMrcFbAue^fjHBMB8de7y#j5$b;W>NaEs3x^ zooLM~5-im}NhZ>DGM#K;S*M47`m(v=-vw176B{U(28a?tGKFph6uuOrQR$I z)Q6_<3tqlxoqL7dZC-xXnVxJ?7Sdu4vqOBn2{Y`}pdko01UjQy_ zDpZ({yV+YNUZ5_ zSRJsd2=!zrtbL%Xn9LD@!5RX&iVZ~w0&5iDD!LPb@K+Y;J4s(Ve^I8fC-3{?_yje%56LBmJ~sB-uLlXRqq1E)GTD(I+Z z9rdiEo|W3xM7XENfqI$%^E4T$JssjHJ+UVV-l+@Rlt48C?KA+|lt45D>of@1lt6O? z(&(ml6%qlN9tn);k${*cf-t4>i(~+%7G5bu&rAbf>Ow0eNo~I{@KU-IlnS_X zXwqRaxXg55naQ9spEaP&y}@K2161j=1yq`ZdzT8NbX?=Vt~LHpU`dmJk}L#LieQrj zlH|c3CD4omB#lEKC6FHmM;eDcN+9{!vWC>sh)G~b2LK_(&B4<_kkSwS>EK6GfsfoU zBrK9R=uv7GdorxW6c{59o+yDbi9#6dK@%lWy&rthROq4!utn*RMbn0nfSGP8T+w8V zi>WY0sSrgS*rDVOkHQn}Lk=a=d>Ck=$pbqLk|-IDXd-48i9r#07#!)#avX-J|E^@3 zBM?NxutLdHhv0_>A%&7D-V1tYBJ9uskVDhpccwrMrNRtNf*3jgawVm2lVF7o)qEz+ z&QvHN0}_%2S0@-2cpnHi^4=qg}9!xb21ZXk< zPzT;8%^LX&ga0IMz5_vj4gma#D>ReAe&VQp6M=pb94L?P6>|$uK<;lKm{9WQd;VfFgj;6nLI! z@A*jF(kL`fZ2JRn&oo$`=(N|o_3jX;wG1Fk~@L zGTnvWP78wgZ0LjXsn0!|QVR(%-4;j#`F(6=4pDnMv-v&7DR~EO#WKcknVJr*hkV0G zsUUp9!PxDQ@hK%oS)pU8ZjWh)%cC#{PXB1)KQR6`jihO&mWm2io9I9v#$ZE5UTASE zreYq3`*1Qk;|S%U(tGQN9N7Z6JMvRD=7#=WME`6@Wu2{*yUs(iCa#08Zv* zK?Pe3wDT*<3w>BaD-w(dejBy+yn^TTtM(hNd1-8Wk(yOoE@*+72D*uFWX`a0MiY77 z8>6^B+G_|jlnbslxQg63x>GzFwYF+1V72aPUjinAvD_u36Craw))9(rH@&seUk!R& zXg-f?Af@Boo^{8wGue}dQ&a6uw{4lhgz8OWgy#E-3I$LvaJ$xKg-KlMw-(dS4BLwB zCB7o;V@XlsrXY~%^JJ^#F&AAp_a)Pe#`?cxUiR1E-Y;*n1)h3~k<;-9YY094d*p5Z zi+ng4=OrIbeumfKxIfCP_O|vTzvYofuD@14oDAi=LOKbnXM{&XV>|Xe!^hA`!KDE8 zSZE~)dSoS~2u)9p`XI=K&lXK-CL=fN&60%44m;k4{T4gYh#MjAgp*xB`<2zuNgF$0 ze0dnQn-O=dP-yJ4fDJ-15qd4+-V~36ka<^afH45C4R~f%sl|%1%C*yp3bOC^yv0B6cP$usg-A zf5Av~YtLGQrOEp`uGSS5O+iEA<=!rT1idsV2z6Qj>ZIV)&o=P%UO}f3rBji*qS=k= znAM}YoTXyy+~2)0_CvjiIRsteMn0;XN0sv-Rn9iArIeLke~?+fjfNI187XMBw)ruh zvCJUXC|DmiZd;AQQ|!9Z&!WD=yM3n^Qjw_ zD>I0ft09`cY&DrSEwof`-T>!0beLqc$Qq0u#P?1eTQ7?VAuPGVd?}XCMtY&;j|f#m z!9boETm~WAR<(25krJbamqSr8E6chvZ7(ovhZb6*gGQt{$7>V#fpv3eb3rq@JJ5xND?L*e5oHZ4ME4{G4=vKFW zuQcnhi}UjP+7j`so8X{HGP>sI+Jbm6*S21O$thY)u4}4Aq0{iVCiyfwqcs@h(l*9= zexu$tlc=v{&Zb6VAB^&CtY+erbdQ3n`l^yk_MWHUSM<8tdD_8l=+Ae6He z&Wun>*zZbKTUtLS`h=P57bZM2DPmdo3~j`^Ne2&3mqcfTQmc5oanTB=A*SU;E#_(g z63vMitPBU}XEAmikx&9@`ZEG{qM^BjaB9ZolAxO>8nWx2kpun0Gm6DwH5Q|STlG0#M=ETsx z7^$9c7U`#-05KenY$M9RL=nwDCg^!;1Xx4RL;&`YK6NB+;=f~)Awz+~!#ZAq!C_wm z93BCy+a;`n=z89dNuDtEulEC*drC%H&=bP*%9R&|;gfDnI38|pf{AQ@&WEWQHmp^_ zZCS?|@gybw9&yhPm06#94)jx$U_!Y+k#=Ve&r<@I^=9(YrG*3{h!RN7UCmtkuaU8= z$8a=DB7wLo2#Fg7$|E!os@#Yzt2Y z7~E!vrg~S*6dJ$g7mphgqq<>DHE0~PZ;?m>Tw_* zQe-AY#*1H~+(5#?nkX`s8>*(rdvJgMoVS)Y{BcYu|2Z-6MXB~05w3$zoP3_H-%d}hZr;6 zLn?Yepv?>NDhNN1sykLbq(#voqKsteO>83MIIS3YBd_(;GzFWHD@bT+zX9g^M!x#U zvZfvt>7r{t0DjX&8g9XEuzww}n|6s>7@P5pCe3(`jN1*{wF#UCVh4*VjFNPV$D))V zzR3SHxm$9(nKr+NFZ`(95n0DL5|_>bVmAZBF;`MugxHw4I%1y@Hnsj z+0bG^UV3P4_|+&~^Kb`VLO*uLi#($h`u0t=Y$Z*+h~_?{6tzfWSx>tXOo76ZRyEI> zf>yRl>F(fpXqV}=K4SAxJ)p}x`%m=`sghM(A{1)Kw%pq&XIQnyu35RVe8+8S5AH5N z)DL|{e{_ECadNj~7vwKHn~@!ijrGGkOdHL%F;mnsft9Qv&G0OjBvd?`-SVtHUy%QY zRT7q+U~5|QHn_<8tZGxLo+|Q`oWWz`?c8ozjw0u|O?Jh^*0BX*{Qz9l|N8sjA_}>L zp~*=r=Q~@rp&c=_F7}p!IZA^(a{$^2ccKTI@nOu2kx*0 z8EMFXSj$ota=p7QD@JpzqSO^|fP)1lq(>AGg!nQ;@n8XJq={b1s*Suuyk~f4P}(VF zt_a;qBN<`HxNY?zau38lUPtg7J01GQf5L@j)sn2Bnu2LPv~)qPWFAtG$WWJEo)CEc$n#>vf!{hX&ABl1$D#q^lXP zCnnNEQ^ySzVr+ul2pYr9f?N>M6p;K8u6YU?2HIs{s~dynS~G^aJMrCI)Ii7U!H5Ih z^zIN|iYu9SHRADkAB9>YXZFL=*`>}$^9mwId%{5zKL|UVN)K=ftNM6-!|sxJ)ah%y zxmU#_XX}c=*x>ep?H0;aTQbrJcFXj%=Uf@DV{SYRKkAkHtx&Y)YQ^;c|F4(wjjXNT z;Ky;H8zCe8`Z=Z6)oZP~wHXfluI1*>)`HM!KxmJ`!OqtZbbuBGy$ye2m9!~aEz#bM0j%89awQ6Yh2hw?ZQJaQ zZ9CbqZ5tiiwy|T|w%JL?cBbZB%o9x2_YCW9{hiSm2;Bo}32S=Y!yo)K`0SqiZ_eMz z7_nyJ#nu{eiplyCFgKOhv+OAtMxlgR^tUTW5I&f;~EKlR@czGNy>?ERzsvowRFk7H!Uvn#{|@qYfKOed?G`*egEi^7n$Gp+2;(!; z(}<@}Go~vQ>^?qTZoCyOx$TTFZ!-N^Y}LPc#M6-6d!)qh=Tu#t z#p+FLZ}dqYw?qHLkr69&ejX0Ndb;NPGF2+wTHaFvHSc1S(KweDYdEE8eGNn^ zILolB>RftUwnb@YJv|I{_uo5>3bgQg z?f0=qIvM_r_k%cTdc{Vlmm9!i|k(b51MRsFD0}e<{mGtvL5Yhut1$gyy#}EJhizeCu7D z3MNPnmv9~AtLXWc&OHx|+h6LM*4@mC*Me)C==^*b`KZ9m6ZGB;7@DiIxh0#oi`3vDUUY9ue2 z_pKVE$@`B=J%}&BBPEWD+%ofCxyJ{(Ik?}pK|w$`rL)-%Bh1R-vy^yrPV5%Y9=KJTmjkmBv;!c8=_RTYad^isf>a|0uK2C5GvX36DsLprG z-D-UV4v(+ATz#!dxeo1rhHd=%eF^qnFi1%UIx?BY%gKhL$s`hVCbuk59L8MQwyu!~ zzMN*THW&pDNwx>XfGOU9`dxDe`6JZp^w^TekbN1xvKpgy^%yf*MkTAGlz1Mu7~ce& zk{6UtE^diwl7ec@2)LDD)<}j|{|5}k;J6ZX@8bEb`OTl*Q zbllt+h-iL>lhG!GRG>)614C$6&nYmFFV|%Mx9Q(-Ov*~FU5E>I5%PnUzS~+*l=K^= z9|b;iui#eVQEsoZ`BVCRhscLkvdc7eeifOpiID!oN)=zmHqyQC$%U~?{QsTV%aQ0oJm989W`cDc+~419Nr+ zJCak04m(csT=c9iU<*!#-sE9iU%$4}9rBZ71h^Rrlx;V1qFyz33&QX4e7CPX4+MXFx2}%#sO}7l1tDztM0V zsJaIEjg}vSf3+>#f~*9aCSQxGPLtha9c`m=^ouJ&H#fb_65GR1tFz=@s!pWZc@DGxdjUe?~UTWZcukmGXPx&V(`WLDkBO6gS5 z0l#`0X&|>HM3|%tQQuYcL?z9<#GN&&g-p_$W!XBgv4fzd4Z)7HuT0`LPNbw`$3`)6 z{-De&%HO^u_+cHN4jlY+}=Q$L23SviDXb4->$3;=<`F1ODFj?(%$KNWg#h29|bv|6ogwO`-Nqs#Ah$ePO=a~AS~36pv^pN3V@*z? zV`Fbv_UNS-V>3oJf7X}L05t;aE>LfTw`y2hN1q4JY(GhbsD|h$#k5|ov#cfiaem$z z=?duX@cg>)Jw_Oe+1PE9!uOx*S$m|m^gVy3nr+ja1G%NzLY_Hqi_-wc5E>H?fI9m8 z6^g4Ue8#~0RDZ~fSYniCU8h)}mcC$xh>%B}08=zvy`#(~?bKLpbnr;!_ZK=S%~q?h zpnZv^R0)v=e;XQFRoD@ynTVOysMQ9Jm29k*5^xBO9pibI8(^xf2z2sBkk`Ih6j zdsKG811!em7RsxA2$i@yZ7F&HSFG;b#Vnzy^tilsqvj97j)qS0HV7O_q<+| zY9XkOZa199S!IAKTKP)cBeXDW_KHvwJ15$4dJZ_r#Z!g#vG*5hcISJi(nsMUgUV#c zj_owaInP=ETBC`2ST$f`8eBBX_XM6SzKJIIM$$AIE7>K=^Zas^UdqbpVsm3<%r(J; z3J9PCZvV~*mPZTDmz9x%;5L_{C-Jda7Boq(d=Zy?h2hJ1;%pFM))X_r8=W?bE%3;r z_fSnZcDC57<81)KkirgpK2lf!71b2-X9{53RR*J~cKlVu74mW!##>WE&Hej!{6wSa zwL8KC$sf*Pd6O|)Xo`a|V)3@chb-MQsN9H#3FKhe$)yX;DMYhLd}4-#j0X&>M#WZY z%u-oRkB4=Gi;K1N@m*RS`d+nh1*8|{B9n`l49nt^{lG-s--Picm<)BGZVqzJ^mZVi zCsX;(Jkj#?ZO;`|5;OxkxMui%!k`$(y-Z+_UAW6cVfho{WCN_}%1)9fM8|e^u8@j<8?!TB-S0-|+ zG_XjG@JqN9d0Zn1`y1Y6L3SuFqL!q4BJ8#-k$wbT#omZcJjtEoPitccF~D`SaGnzt zVej>JC#{Iif<$|GhTxWNBZf!bq| zGUGyFoy;+oiMc`!IulYnXTf!@mIQVsF@D3w2q@ekASMk8N;uFm|4qW~-nDw$_cbp4 z4Z#;^+|JFN#cH}igN6oSUEg4qiGs%3{yzacckGR*{ z)xD-Ic=WZ@V)tR3pEd@;%-l;=iB>= zN%oI9`ORr-4tMm77a<=mdjBxmL9Gf&`(z+_JVHkHMI6x}kjp;Ow|YJY*`4LzU`i8` z4KpSutJrc{6b&4~zfKd|Q{Ak*&rcOG8=iAlPBK-o70!a!`7w|;FRYW4B`OO3%XqgB zIggfl{=KmlNv(|5P#li_d)*0{`kAV(G_BkL5&FE1N)PzGRK1kD$@ArH?$H zRZ&L566xDx>5KJy(VEBaTECyjs+tZzdD7_DS8xfZ{70cqLZ?cd*{MV&4F-Sfki8upo+1LeRDT#wU!T1)fk!`YJb$_* zIvRo^1+$cAy$`7}Pfg;)O=y|+eCF`qa7+5;RBjacxNlfa8-(rJFon%s^i_gCC7qY-nWzuL%jkAPQl^G)ph_hla{|t$HFl z8EmTiwC^`ra~k*)RwB);Wn861*+QrwBRV70TPz>;$+#+O+C-3%P@c}0$`w&-ZUa8! ze?-m1X(qHsfnWF`p^1aUd4jqLKkc4hh@rx1t-l&feXDy5fAfGym)HM1ykBKwJy(8E zSjO4&@RJT6n1BDf3FsqX_JW)k+a_sVkCekFU@()X5!r9F>`)(m7z|mbptXJTR>gNj z=<$ZnTgeCut5KBzF^i0QFD@&}M+Gk&VpqAvm(aNQC&LAZY5jB1Aesf`iH^TeCq8nK z&x%BAoqu^r+1^EsiboNNCEi`q4F_tEjy?riNSql9TL$)V3&Z6MVkH}K2TC}ykOw&xzRu2kYuSt7Qh z0Tl6#lOsh(%a!>We-R78x3Gg7FuGmh8hnpYjn6KR5?;B5ked5XYbBVm*1qMBCC@$B zy=;ms|HvRK@UQ;nVs1VV)9&B7iIX)OH+_+{PNraeU%>czWcv%LtaFx((f*7L zXxqi#(7MH#7N`c~>&7TgDh=G9*P)Dl@hZl&ssb}f1w6by2WZ!}APc1x+O0*n{4b0M z|DEc<0%y5!7EinFD#j6Gtw-sMC@xKB=mfEu>zJht?sPS1ouAW{k~%>Tai2b^GpYCG z&x0l%y8ttTTh4aPjp^$n8POLE4Dda>B!uW%2MS!NcU~J{GD={a)nh6KTQPJhjjbA@ zbjW7dh1J)DWjo_f;(d)b z{Iw+3C`7|rpBO>c^*2$2<`Glq^ipRg6fjOw4HE{@GDI1DAM&tpNwznmGZ9gA+$Alh z>UTGsJu%#362t`)l5yxXd^SSHiL@djj^roEyDQ{#o97CCS@nsNtXZxQuCH$+IG)-y zL%_FgwSUAmV#0|K^)rLRFwPrga)PMAd}_^Ue_1bdOTt^h^3Bs8Qi}e1{LugW0wy|D@Y(%S#vy{AyuVM2nYZ zTeRJD#}eGw9gK{yflHJoLMgyhHiCNh-7HHW+2`bg7DP6qB{vuB7FiFnR@AaGKb2vG z&qjny0!XuC4ewYpdo)mGq%T3s7}k1bIlu5wRES6Ksl2|X5&=;eY71lNi^4jxNHbuo z+oH!;YS7a?oIedI#iZ!tU<IUjJ6?l`=t>Mw619lp^`4b|?0ZI8D8yxb8wjhZ+Dn z4DpcjgrdG+GqTn1T8}1YG-+1<`M7;_ZR8p0kJm7^yjb~+4~>$8ngX=0`B<8kTBB6! zsHWTq>})c^LGrCw=ieHxR9* z?Rl_CxV^mEsHsPhdjTBg@M5YQ5WmKA&hOen8{`Qem^+owpRIL;6g7@HBVpr({3viC z0)K{k7uXc$aWVNRm*z%~(HqlmEwZi1)vaN@%Yu@%MFIU}piJ?ur8rsw^_K>%e?)ph zAMusn!J91$2caBv(jezR-ibKWGt7auyWnLvlD&3qCCc83vc}i2DxCbIoVT+@&)__8 zSgAMXH>{can{j3~aKW)E)dp2iDTqtpDFGoECve*z3fpQ(sD2LYa0Vqe67WV>RE4Ni zC|4F6LiwdNE(M|J!rQjPx;nLX9zHLg4+(`m1?>yh3?8AIJWOL>wZ*SJ?5i23%v%7c zjxg%N5mR=@5fQR+y`MQepPC4V2@RYEwq6?!#`$`AKDk*qjo(o4E)8Eeg3YNwZzq40 zW8JVnt4=S|7rcdMxtB_Si+%PqQ~y}apBC%Tti+QJL{tK51ofcmTFq+E8Y#>ra0DK% z)NM3M%StXZx>`U>32r^OxDI`mC_wmjL-fI%f}i_f=DWpB)%QZ4GW)6hQ(|XHTQrq$ zN3>HyR)b;>7Ef_!q@8c1^zK`PABIvB$s6}eo*8Zucr3>%VX#3N*QXaT0%(!Y*wJD= zhZBb#27AbGo<6e&d3}OG;JV*ttRBc8OaBe&q}A32g5L3X$j4O2lv+QNJ=ypgZgd7u zyDGG+(e?YuiR3%rx%Y?n&(`FF|85^xsUBu!D0_-?2Sv3ErFcxrPzPpnB?o{ z6KD2jQb`?;AkgJm&fPugmgE7h2Fz!J#Exro7it*=zhu_`y(55+yaC6Gt6X#S=W+{_ zWg0#n)+SV%B|5uQ{ddE1wkZV2PMK2s4{p8Kum-nxKUa^SgnxkL^BkdWD&)#tBf(rR ze9dXl%Sw(F-90ShbX(Wk^OIIdpm1faQcV}s%fwa#SB&hlyo$x7@csJc1PB`BXH^Jda?9_{R*&e)sd?SP^t{ zAU}YETLC~683@zp5bCr;RCYCnpShRDkgda!N36_BCHL35WNtrk4tXG_ z8T+_?)NrPK88|S`cWnjh!Z*jqM6aa);xbYUmAW&rmCO!|sc88e9*(xR51cASJT)2o z5&Pbj5}Xb7)8Y_GEg$1}Gww9}Q6}-#$X^W&u=!a=_90!QqzUa6AbPmi18{}-i-RoO zmD}`LLqu=cYapXFxb~cXx24wghMJ~hMGgfGgkcPqFnXqkELIdqncEBmCEkAlD=f~yr+es*`PQ^bi1KI^LULBbPR6Dpm;1aN45M= z@hZ4|Q~7Gllwh^wK{R3dy;Uun2jfi6HK{Lgn4YDwD@D4wkX{=kniEsuBYq32mX0d0 zJ-Du(-QoMLD%vPsnIqR%0#hT|)d|+<*;zU2I@{^bP?;fC;m&+JicQY+l+jYb0Cn`m zAi`9IK=3S_OCfaj+zZg}W@M%kfu_g8#ikNAz{@$!KyF+>*4lVf-Qxj$t#waVf1__K%i z?1TW{X(+XueejjIA!td}zeBSUP#%}qBJ!Kv&cq!P4LU1Gq0xk(R-z?SEU>dvokjsH zhWzBdU140dDKNBDT-tm54mI0XWRVMHBH}q_7GYd}?WDdBPdC}lRY@V61bw*_9DmE| z)H{cmbw3{5PnbHa7ceBF$G4Bq_<^jE;jpRY0vPUTw4041L$~!n-Mux`EM&_fla#k| zQJ7N7#21X*wh&1~4_kUmlu+VK4@94PS_9J{TdAtN7}*fqj=giyN`Qqlf$F~tQ~%5@ zB+qMMA8J)?Nf~A89x-;hjyD(3Y*`08nCXcf#hSmww@hMU@DyZ`n{46A*cAp+DkvQO zv232!b3K`MoG+i*JN#p%DdBewFdPwp(qe~o60>2{RC_>R2=pcLT+<*b9UM^^CR9cd z7tv~p0nC~8wcvH!bfM#3GU;U*AsuQ=%%P|KYPrw2x+M%x2&C>YNsMp-r0~J9w7Y1c z4|d6gAUQCY<)qx&+!MaegUeVNFv7i6MEa-fUW6?gZadE~a;sPny7ez|9+qlP+9F79 zt3o(ioYn|86|wvf9g!Nzg*X39(Qmy5jrZP-viB4QTo$W0A#*upJ$AAFu zD9I=F3~yk}Fv48fe{I1Oc*~iOt84o+IE#*I5AMp=MJa#0JkX>IXQaTv-%q$ZeiA&W zG&jMG1-84#EX3>+<42+Z^9vtr6MEjoR31Xr*;_efYJVybJLXJY$gNe7-ng%rrnjy{ z<%VFx&K!k1R0})=|1_EEyCmub@kSc`#-%Bix+h(q8i9V zRuTt3&Bpv9&F!pYXC7Rcogm#Ay5~*r2S8jhIw4}F3NSF>qBo#Ey;-=XbC|k0Y}5Zi zJt+Z`(f51Q2@U;$ou3joZG0LUVhIZ^p?{d?CG}(cFjg@g*>Q6D@571RkTIQb0MIZ6M;(13iTVunX)?* z7Isg5!v~I^f7j^X&d8s1vpTG?lnJ2iJ#4ctJjt&$r|vz8t|OC zb0;`h4XC;+^ZZhCuOGqnqHcYg!!%2vxRzaFqEx*p34aluBs;i$(IWe|ceJlL#K2t9 zlDHmU#w?V!$SkCJtb?Mxv9WUiv#wzBlH*f&7f!8md7iKDsNJe@Yoz}qbepJ_KNIfS z?+=GcA;bvL|5;%k6Dd;duG?r0x^mD&^IBa-xvRYHzP)C3@KRTrf!!DfZq%CZo=s)& zOF>;_NfIe|UNhnEqZ7v#;i{{u^AJ+;a$`~H0pE{RC_InIPin!~y^Q<>q_Q3Z9M+#}Aly@*z*3u`Xx zJ)e>27{C*0}>QbQ8XFKiXF_=!Cky7Bcm1*0HGAZ7E03Sp5bnagowi%lEs zAi<#n=Y)hZ2LOOx#yyw}`co}xg*$d+qk~?i%&IPfuyBj#;;3yEE5OB$O;0PLF8FH`Mws{y=Wx zPcv#~$~KEYn~dtrEDExd0nd8MUs+2^7a<^OC@_fq5g|r;-5bRi^^~T`-W&N$Y~+Uv zfjPz?%AUhCfZ*iX$CV`MUW)r1XE4b|u-!muXBSh%CUrBGhU5rukFMSq{!?vsBAW+i zB9P3GbZNK*v=SCuTlow}IFw=ewhBk6Zz$!OB95b;4|o2g>vg_J3u^SVN*}Ol_FIu*pagNI1Ea|Tl-bPY=@_@K2J5}IbeNI2QY5avsqTkdYuN_c z4VVnV9$kizvKehE6@3x?#0y(9Gs5Btdn1g3}by}jb4eju;{nly2fUT=Aj!6Ag>;goZsMTf=i zFN$)#xotzmxiRZ#QX#*hfV+haMi;mP+WoF_C;~+S=~>gHUSxR)ocKOZ^ZnjtS0k1V zqQRz9K_1W7p5F@-k?mE9m&KN~&}7v(v6qm+v3#QH@4s*JQe<*n(ozjWqf=v&dDL=H z0=sz-&KAiE5y`@28J-l2mly1JB&P8Y7N znR4MsV@&cB8wMKMPCTJ>?TNEZf2D2tMNn?s8R{Oo;c#?6G-3A3sUImOOnn6{q1%sE z!D#f^tkS&@o_nAPcjFkmnuy0o0ElatELoS1o?YXM1a|yZzb{<83Z+)X zbiaLSYto~pn`?|G(w!%2J%?NA^_|XIt(7#+Zr!~;qVH+@`ofEG85e@sH0REM_bf?o z2ALLpwHnKah2Ud;Wp^R7(SfXW^bX=tP1G)XOrg#={45@WHd9d+ea`Blrz>Q$%eqZ2 zK>n<_A7H{;NX{^{nn#%|tjns{QZWrUp&lJTM0_yBHh|yba?xwFEqQlZz6J`upJxA? zTDD@l;Sojo?h{l;B@%XgxXr&%f($|X@popZ{gUAIj>f{Zkt<8r+Zh{hwCi3K(lPAq zC3{|znkn5=;4!D!R?y(WJbD>!(Kf4bDm~WQxdFdw?K*F)X68ALg2FR*?5hRH6WIkh zRhMty-D7s3Rn=*M!!5r}uM_=Q))~iK&C;yT@n@6bSqR)>ixTKN0CwcBjZ7mTbTS7P z{o0#LicY1*!4n-QAOhKerRS6K&v=8GzNF!RJiMw^gPAAvgT+|b_uKR3_UQrQ%K|c&xxJENcF3`SXwFVnv~rltVdb)2{{rcuwKmJULS8BQx(qh{i!k zStPVp57nhil)lQIyRuN`=GZ%|2HlnBhGs5XmjPm1#<3o| z1$mF6bRM1@1w`MUmG7_qrHpxa_&dNy!o&*Pf@NCov9?~2>>Tei{t%A^q5iwmbu^90 zL#<@nkm1e|A>zu9t?FXLZ)}lTQ2*3W0z1;Slxk=|eT&Op<-?06cDh4-S+y@Q)`*ya z3{$u)hpS`D4!!+NJ24mqvu=Qumihfd1%MX540j9D={{Cd&jt`;C1!Apx+Hb_KOMr1R!&-yn{?7d4#6*& zWM8OTOy>5I2gHDoaPl1l+w14e&hjm}hMM)^;<;(BfdX$}mksd4UflG;DUH`+YzP?R zMxDZ(Fw&L;iO#3lW^SikgHIx2Ls$qe7G7CXUQ+W4&b)3ij?#W?f~vLvXXCQr?PmA; zTlb;9Y<5Ir&{1gD&kwr4XPpk?VFQ_lEzCCz8hgHDaGlL!;9zJO+PMOc^?S^Bn zTGJDHu9iYQjw+U)28ym5`$Mf>Hpb2MY==c*9`V z8+odTOrqDHAxZkEK1c!EgbU->El~`=@1?dng=qlr$>D(F#+n9)&+`f#JG^SWh3XdG zG->{%Yb)UA;xMMV-e{Ml6VOjqnXW;%M|sWqy=>2KSUg2#lIvb{REK`Nlf9KzZa+j0 z+zDQMTE430i+WqdOV&BD4rg7;u;l@JpWVSG+=VG z-&ui6h^<$Zf-sHlyF0;f`bqLoiSWT4n@@|W>QmeGPw0Ghb84Tz@n}7DCf%f^)lDZCRvXXsfWWso;@lc`j2_AfJS?RNszmtVYfCN)J< zX2ed8p`i}!5}=5-YeH(6i9DPC>MysqzhB40KuW5Ain>hVpJ)197`zXlJ=(du-SL^* zv;D!P+wF5(?tpRBI(==Fq~kjB;g@|b`RfCpch_DLxjOD=D=$x0ZWY^QZNPq-;y&*+ zq|G=B{F);HoJ93L17UVd`|jnmG~F9A4JO{0*!GfTw`4+95lc0hI7|otp57K2C>dFX z$%qP-cYLobut-M?67xsKr9X~G1FJdr?|e7-fZP5O=TC-{osIAo+(J;D;3XYuoa$l& z>JZzdCRkSn3ENVO<9wQ&FdG+xe6Qynm|Z;c_icnQ3JA=U_Nb;l8ASg39m_B$rKQ(4a)U*P< zdtaiIwv$OGC{4#u<}{?h?aRjCc(o*G7r4zZ{TY0O=FpqhY>0422{`|`aXrw-S4XC( z>OwtMfSA8-51#PyCDRC03<}{YE^d(3)-%e+&}DxtLuVR1g`@$CGY(mfZ>__wXH`2w zTcF&KrzPnKn12#%LC=*s7(>+SF%pgm2@Q-25khvuMNMXJ zDG6(*EVEt`ZWtMbUi=nbcE{G)8oWdIuP#ZD5Lbj4Ufugt`(+XAb@V#5%AQU( z78#g_=wh=hk8lfkq#xE=+td?*?hSMH=(m*opRk6Is16T~l^9C=AkzczU0rXm?~)Z{ z2-n+E(c6TA#O0%p%a)FvW#Wx_4anhw571aH+U?=AwOGhO=j_|dfDw+>E3fb)6f`!r zyn*7x643U7Rjnr@eB3MAbU2o1tc7cF_FXA z;X}D1&pC!-ykAFhr7#excRU{A$J?OE4^kC5(1R!2-~&NnA4aa% zjYeJju0+U{4_aEeVc<7xJ7kUvP&OMAr=8g3~#!5cPky_meBM1KL80wJnHx_oCYj1b5?cuB6 z3gy?Hfyw$~n`&LHZm%p4&dHV3w1LUEB4odT7T@K1GP-m&zrCqGGb_b364iE^t8%LK zS;=&y{z($Lr&pF8<5ZQGeMPTe(!uUm&|T0xc>acR5y~YbCFDM!?nfMR1-!?^3xzNM zf1({k=CUE0vh+RkzKXx8d28%pUi_#o|Y7S@E@^t0$f~rizdA%Su_&Fkh9d6{&f#Fylh@CLXuaNTcdQxmO;o^U@JvQ z$gHt+n{#H?9;sW5%M@_PtJp4uR`zt=p=-c(7#GY`P`ur}q@BbSC!c7W=Nojes+=b? zF(YjlmZ3qaPQOFqwdDYlq_^!_&queL+q#@}&w1u3)pB-MS=LNk@9)_W3c^Xn`kN51 zBh3b57nMHP_ad#uj{^Lp_Tiz%Jz__s6Nh7w6%p6e&%E=Fl3ZJ z%=VjwC5xQSOeCOWT^ZQg@!{j~p+;XbyihzXGu2qGdA#W|x$@zmyVUzwSIjb3lLVhtyJ&?qfv$q?sr$ zYmRAvAxGO61DjwPj)lIoPARGEbSAowN@_K!Y;1E~$%250^%m4jg&L3|$utc|tC}ME zC@o{YGfB+59}PX187R$87r?YZ#%UWxs-2?SVb#YVD@U^1uGZMzT6fUNVH5{s7H7Geacb(xvM2$ zqRwA4+32rs?y677T50DF?cJ6?O#hReposH-au%p>+>xo`tRAMn|4|5doe`#<`*HSu z|B9LD`~KoC7nVB+@c;T`p8GlfzRNzFQ&jBx+1g^hv9J4{&Hs7uQLewfGyD%LiK#Iuha3 zJ5RF$Ih<2@$3ra*zq3dbGc8{3A0g~{$}d_T-BL~2{ErUYOgT;K|H*j9D1F8#bq}QB zRNM46Ss{bAxa-IgY5h@fvYxbUh}AdCtO@H)eqkU3DK(W{F91PN)!+IR%LrP7oPGQU ze^BCI4nC)Ugy5+G;sBxetn*LAIHP$-6jyO>61+-dZ2ESZbiphCHJ|P)PuiYytrn|f zWn^@jturRghz9R=@e~absHE3%G-_*#TVitb*U)>hDX!3;AEv+Z^wuzC>&T?F#VyBJ zKBqXo(S9^ukl9~p0F~n|oH{N`4l?xip(1$(Fa0;QCup79oPVwHFhfTG*Z76WOL-T3 zRZbfPOKpuLbJ?=synM=~`x zwx+dr=BespQgo!cFZWyuXKKrS40&76kEp;EWL9&fW&Rs^Jzjo~tu#e>Q zZc?E~C{&q8^f(*LxsHm^w6CHcamIWgI|%AyO$p}HQvSqBbk*74P*M*3jxYwUtPX5? z9gb#{;oi$GUQTp1gm;B4n$BCE#CWj6O&sl8Xez$0B(aRx-hvH5blx|Kj){r2;5vp| zJM9T*doJoJ{uW6xzye2{Tc-ZGD@OHmZlw+-h(uPz*lYZ+rGOD)HZ1ZOysQtTLt~Uq zD^nbM-Vjw|_*b1iW(_Dt1YO+KF#ous&^R~Q@~ z8eTZkqkTb3Cz)_zt`79O+zdKVdb5>!dO(UV!Sz!iRhK@5gKKBH$%y8Q|P)f)|Yaa7Btz7!o4@hHx>My}kq7xWhLuvjhz69<*E$5l1Q(exg* zT0FoWR-U(LCbTwkf$KOVN3J_+-Aabp!Cco(|JIavbVlk{6LcY*4=m1opSEz3{=cMK z*gBp0X-3CkdvOhS-!W4T%NaW~DAjYBv$O0POmmD+JDCeXS)MRwwqruc{>WIyj@FBOEI9X!+ec$=Q|0wKjie1)K0fF0)v2D` zBDSK4u`R3AD#jvuGrhYx0&Ce(?EW>yxV~-JmrResBNBmvOc1mJ|%Y^ZGOA*#Ib))y+%@;36`&u z21pII^^%zrm%n*VQkQA`4JI*9LW1D*5q4H%?3E56@!2!*6(h8MxoAUD`r8x(b^Pwp zfBkld6w62EaP1slmXZt7pDo2PI{u?wl5r9yJ0{77G7@^Y4}oB!%@B&LDen&wtv-~t zQGXoFPr}md=V(GpTqF5G0qytfoS-t2TF`8=zO=bJ|BZQHRscV^PkhF!dD?gU{7ipa zvOIVJ7SasT3po|wCqc^=_BBVx)=9B{vrjlEehnfT^6vB;wP56SvE;zAZaF%>lVtJ2 z9rPfkamtPQiD5S^?a6GNzhS&nf`b(7u^o2;(X7iad&^ywGW|+}!g2}O;LN$Eg|xQ* zT?}qa?QcXQ`LsP~|8#}!n&AckWWu($eu5PsbA<|svA_1L4oOD>i5Kf%pY>v4a zo~0CL9A|#FP?Hu4KX7(&u4RtFh?_TevGY_b7Z2DoR35~N=%shcwBkV z$e1~DEgo|`lVkGChQ=myynn{~OWzN#nrhQalv{k?2nFVgLSdM-q>;h)t(g3LM)88Y z+mCAXx5HfQI4Pqo&>SLrrcEk@cCKSKmjtnCN6!5FYk&3oG$gaEdZs5wq5F7Ylf07c z0nq6)eM(qvBFT&Uoog<%P``pX;hv&YX*ZSQ0O2KXxR%U9iP(XARWlov?EtFtrqyk! z{as(Fj^P_h1!I+6{64RZIwpJh;R-d9dYPXX^g?|lBbm9Fs2^~%9~#jE&%>eP_iWfg zonss(NlEH;pvU!WX*k*+79B_?#JRv--32A#f1d5dFMqEuZLT{Da+YJ?c`*1kQ<*qE zd|~c%ec(M!1-&5Sff3G~IyxaULZbgwr{Vc{UJvwU+sCGSMtB}F8?bZ+@V+?*nHD0) zzV?8v2zS~T{QbY5!I{b&SutCq{)r4Q_=B$r&hR(>qVJ$YSOGcpN4W$0TVrI7Ttb#V z;`YtJLjBF1HyE#lO6TofT7ZG-RI9$p7JXJXb*^IBK;oc!9)#Utu2U#nL+8UW$MH6^ zNv`7;b=JB{Tt#w*+F|6~u7kYweq`P3o6!rv5%E5KqjXC;fX5kQ^?!y8f}{$lEl$!h z3ssna3Sw6E>+|mr@YbN3a0#GrFjdGj-j-YX_0F^wNR2I#cuBk64TvETQxU1AH8HJE zS5LXjVm~ERqYy9kXXl52JFfvBudspTiKA6xfr=1gfqyGO!rX)R7`Oxvo>yGHtP#wz z5CmD5c-P6uLmr2ElrEgq4sYfK#TmSdqtVkmUf0$_)i2YO9a2nmV}Gl0f>hs&>%GAs zr1uCI*-lSVVF7KwC|r=CMpKp)(UU+hM@AE8Tf{Us)r(&Bl*&{a*@arIGf$=IdG(qu+L;E*2EyLkl*!vztb$ip7t<#eb4HDHqxr_9{R{_rn?$ z)XjRJylg4v&=4KaXzi2VezV^ShF#_~qgd7w2f# zjdoF)5_R~>52kspf39s3*263tu5~ZQqQ=MtUXoUaf5`$F96m|~v!;cWzI58hBhDR% zs^S-#HZFN>_dj!IX5(_#JN(yLR&o$&z~=j9P7JprEo!ft>&n@UShN`{9z6~mI&qar z!8}i$EaH?wyrW$hPg7zc0rGu=?Lu44h@@&TdtZdr(*lod)ZULOTYIGCSR=_7ia!g3 z=oK|LW#J-np<~=e%UC+#{(i5YfClT@xuDi~ql-syZnSH7JmE@MzE23PnXB9V;jR4ss}Drc(>{xE+{PE@O%P;-EuyL8% zR`ROGf&6n-!*Mz(z(*rrFHQ|b5pg>mb4Jg10!gF88kf9v%cUlx(1}MlbU{l)M~dB5 znNzo8jI`Hjly#J5^H(qLOVpMqjQA5Q{~fJX~IY>tc;Pm8CyirS<>7Wy^j0qlDoBfUP{>2yC7Hm6u%_B>)n zq|X0gZ5vRAYtd?-^!=+7!zogo|Gw*RX7AIK6yl*RZVC!{XZkNQJ3{FErc zD;?!#v0%{!7J*T?5J}_z)Z+(15#R2ZsZO7A()nx>rew6LG9^K|-FUMxH5A$;Z^{EGQ8$LDAONyOgfRuu@2#Q}ihV_>ptC_own Z`01yge)@m;UjP6A|Nko6O9}w^1OSSS$<+V= literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..c6a8d273e0 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 104.1.4+up57.0.3 diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/README.md b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..13fc35d213ce510ffa700fdd9e51e031434f5c15 GIT binary patch literal 308608 zcmb@tW0Ymf*5{qJZB*JurES})%&fF++pI*TZQHg{Y1@@{?>gr^{odRC_S5%6?=kjT zYmBwVjJfuXi2t0w89^8c1@zbF^Ssm2b+4(Baeew3-HlvxoTWVFJWa>r)N!@r(rxZ# zRHws=!o5A6Jw!r6NEU?_Xe>V~^L8ye7*|w^6nJEAVCwvx_~zIC;gI!Ktt<1ePDFlC z@h-<}z#ibUg-@9A>3d)J@p%);F!2fhcKD3h{kmIzG~LblaT>XW$N#ze6v^Nl*tL!; z3SSa5@WwGb++rNXV0Yyxw0q(N6pI`7`bpwH^iuvAWhd4$VekJvh@#@hTpAXGeb&Yi zuE$3ATb<*3cK6fo=Uawuv+B>tBYob_4ci&Ww1HT!cYaYvDrtKMr-7RdH?f0`bQPiM zPSRjJ3R-^ZujApFiBeG-lAI|~t{*n`JRcj8H#f-cvA4`5y3fREd z)NX<@*ij^O8ah_7%Xx+u`_cY`g?_8b43{{QLirY0 zFj6zwaP$6w0xCQxc^~W~RXQ{74mIf9$-+;l@r`+whZJ} z*yE7TRk8NvCgn~fZB!2Vg*N&XJuEQ+RS?FJNGmMf0}3-t_CpfotC)Ha4XaHOGz~_O z^nhfaDfqi%8{zz!EXsdW3E!ty<;?k`l$ zZ$fwtX35fx#KHR)_e;fQ2H*?CLMlX7`fT|cvJEfE=|9KS)t5)SwHSt8&d=QSJ1Nzq zbI7AGl=~*X$bB7WlT#`n%MV`olwg*MxIe*MBrR5|eC?+GQlM4N_7D=(%K>h_91OS_@k;%`JPk!YCe6 z(oD5{UJ8xMJPbS5;#n5nbjW}+jrDGBK2?Mv8;ToSZD`XOr4dhuOg|ol;yp!sl*Sx$ zdB~b(X~hw|m&ZZbkD5@KE$BbeO`X{qsKvi2+IbLfQyin+(x#dG5n||-s@nA2+(@Rd zE9$+{l@~I)X!*^Fm*P;|Nxa^Q?TIYG3Y5YjG5%ZH#0Hb26JgjQ<1p@((x5-bs1C`U z7h6$!`>xSX8`2|3$~hRLrw@9T_$IXQwFlpyzyD@8S;TUwc9=$x>79Kp;^G)$w<@pz!ejX z*)w;78)l@xVH82q%q0jK@sZ3;L~ib%b`E)!!$;Xdf0-{$P~>g#P+zVR=SW@g0&|Q> z{VD$=w?h$^!7TkQF(R=Vfu07+2=5?jnOk`JryBZk2%fILuh!jHhccosgGp58#h9hO zN{167gr7Mw;g}3V!!D6qDE?A%ChktudkmT-8bF+TTq;-Fb%p$ zU2*Bciy4+~We!>$^^2MB}``AWZ_`~5% z9b)^PN@R`p*DAjwcyROUJpzB4iPbIg@riKU$G4el2qqIzQ(r_^j4DR$r(F0L5tna(Ka4cOcG*9NNi zb&0e{HOY^9yFbhWJMXvCGE>5O9b~SqpW02}?*!nA{y#FKBlM+(3qv90tC##}%qvC{Kr2 zI(ohfy?R|tw$=Jk1%5>gM;6uNwICbGB;J8@77A)JQfs z71=07(B}lo=)xy#6Jxi8q><0l+U@KurmyeA-C1I>y!^xG&+pm1ug@%7J)m?UGxLQa z<`X+gyRE{&zlLYaZmRm2Lh3DUQz=p1BoIsZb#gHECtB;2egmhr_)9<+ORS5H=~~js zwbWG2pQZ;P;NqsN4iUS(H18exey-gdKLXmhoyYX$Ch-1v{5~!B;llGW_wi7lo&B*l zh)FR#yh#^g)P(Ku+ors=#lg01g%Nf=GZrZ6cy@9n#$+F#xLK+>l5>Rrm-_1Wl@Axf z;nwgreKqO?A+kaoXJMr}5M5Q#h*T}v?&=?`B?wvYuw2~&p8aT9iME`J$GZdRB@x+< zYHzeOUwpbq%moLg9rWyPZj*qGXHt52UFmQ=b|hr23n z9&Dv#y|2eQpP1gSrazBPmMJI>ixGKNx|2e~5`$*IPqYr@rWM}bn)H53N9L#@rVBn4 zl@I-*8~WVD2ECSiPl^b-Ss)z``YN+dIXzK+A(Fi*AQF|oUm>Z&GG2R8sHJdDfR7UU zI4)3&dZ%0`_4Oiv@d76Gu6=x&iK}!DSE4VR2kAJ4R@RT&FnFMW(IWUK@buRXlEkmy znvVew?LnxiASJ-yVmqGDNhvMl(jw~7zMxcF;p8Zp*(V?oACruf>AZa4<5**!_#oudg_z?}5He)B) zk;FZO@x9AFF&X8TuckJC=FGD;Vw)xt%~K02OVz1F;YtpMa@8){`jDv1-kt4)!A4}5 z7Zq3Y_gaBl_xA)r&-{cno`h~=_Vl+GDdamQ_NkIOURNg}d6P0Q)zcKj`{u}r*&84 zAz1O5d&?LRTTI(k0GYDK*aVl&}uhh@6;(PiWZm5(dcaL2~K!f`%nvU+Xn$ zY0+XQL}PGQe}uG;rD?ElJXlaw^tg(0W*N+>GV*wC=!i?n*d1%)d@+V8R&q42eAJ0sEi zk;GXj;!}p9sAQd2~%z$GVb z;jCb%?NTRFz|iz&UK3^q?L>3oB1dxC!w42UDAAxeJNRb9pfGl@krftVN6$W^$^OVZ zW*!{~N7k>jXOS@8vXUA@X7Awsf!a*m)a7>!@u73{MV`)yOF{!cJt=*ncY|bg;uv(a*Q7=Upnuryppa|i z1&w)wjU}02x3`m09at!fNrl<2+H%~z4%Gz*rAMLFG{3>gl21H6zdnX>zvUW!)Tn-w z+U^L$yJ=UxSHgd{63J;$NraOPmkHSQQzPdR&u(4BXB--#%qqI`;$}$qbjM7=-5ZgZ z=V;O4*wd=_utV}_nx1|z-6xA#pu69n$}Q5^y#x{SR)9w77Q`Ef#TiU*E|JeKaSpR{ zn00}Gk6@SJ2qTz*&Wp>_r(P+B_GFTqO@W|Z^^x6!S4GPcUVvnAxGSH?tF%^YNZyFZ zvsOnl>CUsZ(r5t9&I3U z#o(dmT)l?E)VFbx4??aR;qmA$KASB>pkyB99-AIQF!ct7(`g;mMXOnzwm&FN8B1UU zT1YMsh`1O?iqGI_D;cx92k*670jI}YYO+mV6=}t20vPso`VfSuy(i){S)lG9<_0!T9o-rzD+k%YC7DfAr`AGn*B8;5Y;Mi02RTH;@{qy%Z&+ll~PtTPT%vWc^1QMzm(V;RyI_;}OXD|1QhbBd) zS=UwhkZzIlzm;~Do1?QWlQ;7odRxNw9jhHGoHQWZg8Fq2OZl%6hilMBp&sV(frSKjWD_89CcZJ&qwD z>A)nZ<*IDVpF3lutPFiJ6?3IqR#F;eSwtLLpv87#zSr(`?fl8QPm{OD64gcIRN zZW#00icA)gznPVH=bcGdG!VejYZm(TK|^UI;`LB^>w;~yn_g9Z)epDOi&3Oem2`=? z^--5{uTV!d#o9m>emIhrPPlLc;;cbhsF_DJKJwKB%8if?v#u50ebC(R62M$_Yaz4h zxajk{ZDsYhB-}cB18aJqa~p^XE7Br6y`Y2x6#pnIw@mm(lDnor}>ZHIfd?N@tJ7M z293M8Fh0^LTW7>ot4n;HWaGlh;~&@wswXVddw56w^taAm@YfLXAP4p3BYsPrw1yJ| zryc#W7eT@}U4pYWi3flI?bkQH zj1R!we?NGcw7ej5g0$dT`om{@RKOkF_Mq)m~xBC~o8S zPa%6TG7WJVQJ(FCEsO9&BN^%T;RC&r1IS-(V##3*A`1+|7KJN(kRB3{Is(yZW4Uy? z>laEOSfb3>o+*D27l;67Z1@e2n-Y_q8nTl>x36KkS5L41f}M?NZl8ywcH{7aL8#k7 z9)nQm6$F0-F4^ue^`ArloPm>gM9{r5)@)u|PB7}#kU7@#Y?@CzErG}Dk1l0qnyLwN ztg|6=thdO$NUdLuZ1M&-g<6}=XRHxc!AKd;Ebma0`cb&~qag2#TUQde=n>+CqgDCp{ z{wL&+ualYq7nCB%<2MvUy*~izXpumtRx_3k!u(;X6^(}CSgOKOSRG`&jZPag%%Jz? zazwwbDm-18iIHX#*k%!bVw~LyhHNO8;c1fL%AAfb-rq??Sd@a#Db#}u5`$zH)Ij;{ zH)Z%IAl67DzIcH!J*?}koPAM*h#`{FrxykRH0u7F9{1? z0wmld$~q>$MGVP)pnmKnGIqR{Lix`OP=Up-N>@r$Jl6eE5CjQnE-GkX&1d4u&Vx>L z{F!8ls0Kx$rAT>0H=8PhLKc$f*NIFaHlEY|8)3;~p(WLIw$gLb{!qbH*DRpORS@VD ztOPX>=u`xVj8NfcNkEk!m&V@UEs>z zn4x;Wg=TBg+M4QW*vho&U7J4}yXJMdO>}3KoqaJm5~!!r3)ESn26Q3GzGK)(;jgFC zsUpZwCO%-0i>dmLYTV_W7MZ#6W)kg3f1G%G;orBXdZ-%mpJ{VEk8@-g?f+G!@6 z%b`4RWjWZ06%g%$OJbtG?s;}}#SuZM2W^mA$1Q5j)*@=5)<3o%2+47H1c|TDuEG4* zx>Kp`UirU3z}kHNqlk8|uFCh@8y|ws_KIHC>pjG0Fl}o~!=eUue!WaGtRp!{PtGGm zjGP{&Ac1?xOd5=weZW87#mCNjw~qn8E-v1h3b2@G8CX=4#&}pn53ccR8Axadt9eu5 z36Qx$90a(zf-v9$ILT`UfnX;&WENvF1az4wV*IkWPjfr)@jU8HQ0t5M1Dua=P!Efd z7KC&{Dt1cN-1l7kfYJ#b-10<}_e~qg*1z(yP2T^N`3jMy4~&>RurOZu?UT$AxYFPQ;5G znD?19%S!Lp9I_yKD8glO0o55_Rj{2P7Gk8S7xSf+nxejDp=8Bb+$JcA$Gb~+wFqn~ z%9Wytk9MUu&CYhc@7<8%GL>GTTm>3ko-Sd_ddNzSZFN-*%8p!>G&4pn1qJzW;+4So z-lyaz6t}ONS3%G;nP#gb1&7ipM_ABGMlocP_eC>{=qPJ-hB0Jq`JnA%ekh90Z0_G< z&uXtkNP5WLmnee1MT)l$-bbC^?dn9Xl{m-~WNHkgc)-j|=a2LFgEW9B_;~=K<>;q@ z6r`0tiuDZ{vSZ51sAp?&(QKO8?w^;m%fF8AkY-hW?8K1Lp`?2-kbx}iI31nT91D^sK_Dg60<4cn;5b_$3R1Q2wL+>h(S>WO$YeR1t@B*{~g}_fB z=R8s9hTZ^njh;whUD_!)?DLVzL29QY-20o)23KX#272-sBv zXW*n_|KF?%wFamJYJ(t@^7GQ@NyYAfsBvsQ2$b8~^z{4E}&>#fVSHVA_ zI+BBhB8YX3+TZRyngedZ{$sD9_cut(`(F_)AP&tUvU`100=M}D%uT%)ga|$7{7R;X zHlF~C9czTE2Ehu@5Rh1SM1kzvhNJ)8K>N$OAP!oA;9m!D1Hc=^BOVqhE6ZG1=pL&j zJffJe78fnhd3-!GNge*!6G0(Z_!#5f4j83uJGElhj#TgLMir8ha}E#zW5RquPl^X0 zz0;N1xB}5-)J_*j9lB*wn}BX}^9iHZ!^?}DMw}hQ3~d|e9ZcoM#-hkG7L{_7xo}tx zF}bgMpN*2nPg1K-(#0gSeKc!zQem}-Uru_KM-^_-NR|}8f}!)ZHS=%`vyl-c9hxoU zOTgU2YsbM%vAn8-qQFGA|;L$n2Hq&Wfm=lV$YEqpwk<{Ym38Ga} zaUV$)lR(LYRG_c7heKfqluAO>Z^zL(2`> zJ9y!CCgyB!tA4i7@QAQztIz;l-pKeP+s+|lc6aH(v%Ho%ebRzsaK>iUVE04K1b^ZF zep<;4DFvl`{=oHD6`~2ps;X*{21@0`U{vYs=YoM9PQj8iPOJxnN$QA;ETx+Dw_`NL zaJwOBxzwgQ@L!q7C z@B-M%&Ds_ErVj)zh!i(5H?7eFcDj4|h4JsnI+wY{dePiDrJFq}OJ+ll=Vj{h3pT5# zWY`!?kjGl>*ELAub7j~TGzo)M>B!iR25_m&A0m~&S1GzUO1rC)_l|cz9dCmU$Y^%S z&$QH258Kq_-hCz~ij)&50U%ZgB|D~A`d-nWJaZ#qBkEXvo`9x=CgsfYfQ=xR2E%DE zg7kdWbuist&7FB^iEEBbldM4T>3+Q7M=3Tt*Rze2+_4Fudym6-Xnyspp$XC@KIZ~M za)HkQ^UxGzD2MmcQ}dP=W+=z`GYCQcj7F^_0S2056L0{7V7lIoz2V7f&+|`{z03PeC$VpYj6o|$yVFERQ1W0-^z8?+?K+>br{z&?e%s-O8 z@Z$s^I?F%&_0+8W;!uM!iT1&Wi!sP%#x21Aa?d?GL{F^~(8OQ%7-$&$)(8P22)!B< zdH@a5?8~6K{oo52+v^`YKocJ%CJ);WS4F42?eDp%s!+*)L=TwR43r=R4kR!?m70;0 z0FZ_TlHmER_!dkMz{LOtZCfw!AM5D<3`0R26(GPHpg$X&NDX8kxry;OU&B`{_Bi9? zln8CZsGq6OZdB%~`mD0>XlJ(R`n0WBnO(m>oz7$82fE6V2VuuSF_qwudSpe%mvF)= z;Vj`qpS&DfoN<{JT_fokNMedK6Tz+~$?4A|*4$zUJIWQ!ek(`y>1y|S&!${%ys$t) zR*O({D8K8-0XA)D*?4XNlg|rv9!zWt>L3^mqEuz*SPn0&S1j~y?q9V}zDVEeC>p%* z#)`3Vzvg_Nv^b`5`PL4K(d=Vgz7cSNY5H|J!iM+ca2eH;c03|gCT^x2?-ZTQW4?l+ zS5!9?BJw2`(lng_g8v1ULV4J#yWQ&oq4sPbR|e5Dgz(ONzapXGhxb|ix)ChmV#Wpw zo|Xi2eQ5&Byz)dKz91>G0_f)yT3ao) zFrDh&Aq^f*VQQ@V1&ZGWm*sj5y4HEusD#EsG{@Xl`Z;AXHxNOQ-r|78>^P72akIj=yAkFdp>I z=DD>m3URZ5%1QRSzz`rPh$p)P=|Dq1xLpE^gJcS$fG5fl;Hd-+qaYm#5NQhPuB$#T z9M(H%H*4wDoRAfw`@TUZJs9IEFiD@Dre;(ncdyndbJ6~4X0rOlvL?~|{d^{9vp&A` zz0c<5N0A5>a(u?f{nF6Gy@z<^Bke;gaL9Xrf2dyv163?`pZ+iWF%2eRX|)@S&f^%< zG__%~9ovoKWGp$-;zN*I;$SL+MmF1+V(_{5FGU+Gl_ZRs_=F3s&F&7hnn?ov^vCI}id_eO<-5u7>b` zpd;{~tb$G|$Cdbw9_(+>ID*@aq6gOrs}9r&YYDLP4mk4yz8t_Fm@c4q^g(}2*TCi! zVJ)BqGP7h{*NjaF$kytW_GV2#8aTL^;-N z0Xf!%>V)l-7yd6-0rp&TF?%i$ZG5l6to!4RSZ_ct@;|x^Ae4a}=i}2LU zx{=jG%!g!Q(sZ&)UFh?G5fYs_dHh3Nh0*YRCen}K4Py4>0TrH-$e@g9YMIXOf-qYr zddbpea^13yKm#mRqZNnTg)$kuw`6 zRw`;+3ad@EQ4-6gu4FADr>u093z};TU&U;G?a4^)D3)lLg4N=aFlJ>*P@umF4LP@B z-ELjfKN?r#3GvD_u#B?7*QL&t6 zh6K`MUm!LULm(DU&IYy1d`Wfm1M_fihZgy#^dsU{yhWYE-7x<#IkLUZ>}F03ci6mFI8i{_ycIa+zO8q)0!dM5nQotsgXK;y#r6fv)5_etj+xkmxSL z-=1SxlQHgGHfl+aDHecHAJuUD$il_OQwY;G>pZiirKU4it5;&*=TKjv_Labp%AQ$0 zz4U3p>w|(MutSBCj(Qt3QA(&8`sCZQ9MlWaOif*LxU})$YNfhszjghi|s^aD+=VaAd{#oW9lLoQ`bP5>{ZVv zM!0~`(>}%_+ipl<hcu~LM(Qy*2FVDR-+Hj>{r4Ab#UeAB{$!HLUAZ@} zqK6n=Sps{Wxc%Hi$4j7a>Y2jo^uLbRi*u2!(9UDSpm2pIe>_t079d)R%3voNb42Ok z$Hy^SUnH%cokh0s%gjG^(y-xkymE@4s#g-sn912mJ(&u&#__Vvq2!=iw98CY>_VE#W2w9+#;@ z4d;JbKaj&{D;skxf0XO>h|;%=fMd39e^HN2Ky-_zS*Yx2a*&_GxldZlRLX}tz={aR z*7k?_KbEjgCAyfaPHAvoKyg;mTAl6lYEFGB`T3yx+q2uhLCIzcA(eJ6L=1Q=uEC>| z8n{_XNlCNdS zxl|Aqclnd0K>;gJ5VUhCLiaUWi+k_sZGtkml>AB8AoTD+dLjuq7-E=&8mOEmZ6uMk zgsIq{2fV%v5G!*>c5dx}BFUgmF7yxXAdH9V6R!r>1ilM%emgDg^JuSyjyL79o za|Zl?mroN131Zn2B&bTIJ}?^dW>Bc=2v*q=ATd-u1C%;~%kKY)M->{oGRi{r8oO3h zp9w-D1!qh;z!2~sgMKbuppP>W@1sHd(ZhKnfe|Xd1VpS43Q;X)tr6IX+RvCI;LM_h zidRwA?#NAJ%kys8E7@BYHMi5&2mTZo(D*tRgGX%&S` z8ao*#l&nOB>vI;{UFZ92J(cL?>V}9*O>HuCiek=$WN(ep-gfcr`>TLYwoHhOG?v5H zRq5+n>zP^{LSJ!t_%(CVG2p)Eo~zQgvUcp-$ib`;lIr*}b`qpp7e6fdLa%@H0@-|` z`%##o5~NT7=vJqS|Jo;jO4+@J{++4^u61XWL0&yj7$(avCEH{~Ei!VG3X@S3{g}!O zsuKQxRV%`7+GG}HO^vJ}G+O?hLfb5ta5`mGhpF9{s>PwaDpmJSn!bP}07wtW2pQqW z!1A_iTyLJZJ^5!uy6=Baro}nmCtME{P|~+&!+iHgoC`v)c6g9)OEhYP$>2yNK~s5bsN?wBuxfv_oX71=KNiJ-jw1Y3I!Vn+Y&J?-gRJm7Oxiy}`7aoXt zORe)Fe*uk~QpBalIwWo9POtWq*2Urnk;V=$PxQI#-*=%j_s0afg8WxYM7NFs<)1em1=JC^>MI7RaAlC5GYCax69HTQY6s&>Pq2$%4QvK7yc? zg$msx^b~{ep5((BXAv@ z`_cN)3?V}IG=4(&Yy!D2K7ca;;F|#yeA*5SvnZVFL8-C@Zs(6iAJ_LY55Q?$Bep@z z8C323?*RR&MF8iYfDlsxPvM z4!VY-W?0ZJnW-k`s-!9cDJ0E@?URhaSa22oeL9Jr2kyb<^Rb9&fw-@6+GZuKmYakQ zStVDuN~iL}!Dvr9|=&7*AP)sCKV`W!nod#X1rKcZO+BarM!7s zdV_Q_OAJ{U${w8BPtLSK%&qPP1~!vW!Ea@grY$YJa0`x;p5>-UrKQ(vWi9a&OmrZF zf)gW}Vsh?xGn3kg9z-#?{+$m^kb~Q&i{sB_}!#fD5ezBjPOxQ?ctKI^qSLv&3(6y?V_Mqf-<AoXn(WtI^QYvha^t>=I+x}nU}LEcJ#gf)@rwnA`Ihq`ObAt zZW=}coVQGAZwG{ z$_fCN{x5G`fE$wU|8p)q7V;l+=|8`P(7$r&rn-2u!e(ZJ308mn_Y0L!=~9eDfc_g9#Sj!-GWj+SpgdvmNX(#tVKuu0-_xvo>LO zklvTz?|KL|!r%3f)-|k0v7B$;eo73d1RM+ANbV~(1!iR&EBL3pHVOd-+qPI;&|%necZ?c!-)_Oj)(UP`2ygu!twvcVTF;+C_(;} z!{S6#0O3R|0KtpC0OLfs0j{k;SkX=(0FV7;9J5E%x8)eom7inBaN|8F5)fteivB5I z?BxB|f*hSY05#3kzY7id^6wlL;MR-+z#F&hkdQB(fF=OWVhRW&|K}VwSK>eBuz##W z{VRtxqeOYmvF^6Npc7lhD0v}7-IaoyxCvn1qz;M9>gwR>=JR!bS#_Q9T3CDATKQEg40ooF_cP#tmB4sy*iuCCf3(q(rz|1 z!C!tGvXiybYlahWk{P;>WA$$-Fv_fXGrM^kaUdgsD|E6*l2#tqvff)* zID$J0>HeO;l)8q>4L%I~g|qEXJmMp4x?6}yyY)ovp*-74K$ zuaLlPF3a9}GWMVI%Dwae(kp34a9&;p(gEB40tIS^>=y}iS|z;UT+-f!py1m1SHuMc z`X}!4YdiaUQo97-9t2Jj*lga21oc$!MZm3xo)^Mg3iAQ#(T~Laf0vba_>2Jr?wyZxLDv@aZ@v!*xf&_`+sCUwIS;KoQp>w%DM<4tfGBQ1ApR3XHTwcAZeAcd zFAJA|1%kpM#0lbF*BB#V;Q`Wz-kUHiS}eSOvM&QWCjfkeNBGM+@!#+dKmb7)1vc(S zAa9SQ7Mfhx5=a?XXhVxhTRJ$$r>DP!aq>HQ)oB)M8$MT48q7= zJ~N_^^9z(Z@XKSZIG=eNaIp0qMQzw9ldX5PWpnM&O;#c7ct{-U)erGMww>#%^u9+dud3j}q!v}{R*1Q3+c7vPlA z@Ft}kz++0wKqjTjfOEeyQwu%24C^`(l)wHkJV*=uk~F#wrYAspsv?8`mY#m+T7dMt zI5Sa7Z~U9|n3Spm%;eHGKAHn=!78NSj48PSClx;Y$I>&V6#Ab^&tKMs|5bYG762vD zr_L_ftDWnMzJ({=$#Oh&RX$=Jebxoa^8=rYc^YU}6KCFUbtbvP$Dv=TW^Rc4G!K!C zCZ7{3q$Ig`(v*iB0sV+>|A+x1d%`@ogkNwN_*NWNGK8jMv%ZUUugz=|l9DA_;b6RC z6={UrSt@pxbWOLTz0C#M)S`3vxM(+Ra`%Du&V!AsU>`u`O`|r^wSKeJ5T934S8X}F zCa6a-8#i?fSCW%DjPV?B_Du~8bv(*G4;6SIy%u}5OGa1SA@|6TlzCMwUSAU-lwTuP zy+KfQdRCm)7AD{lyY?lma+*B!mzQ{!Od=O0h)p=Pz~HDyk|>x&s($TPon%CxuhxQ~ z5m+lt;65pco2Xqcitj_*FGR=UzRJI@BR6k}4sdq}b6;=f2uRP|Xb|gRQIgQ-NbIQ( zm2N|{w=J5dIHsRs8;(iHCCCJT>foI9r0BobB!Vs3yG2>BQ@E;%gPgvG}VU6JA+ zgzrrK3gRl<=Z3v_DK(p`C)7~y_C#+M5W)wT{=IW@V?o1s1<#?4aU&y}S93`zg*#(0|%b|FSOdul+QHFCKB#g$udSM4n+vHzK~auEvggAsG&e?ps$P zNUICj@}`4_{}X{?D8VQ_V=6&7E~nLKRV~Fb8P>dOz%_nOo2z3maTEJoLfalXjGAX_r|R6Q#`kOc?IZ@U|SvBxyV(%Z1y7Bk~WbLj*C1SJZ4{3%-nM%#y zZL=ova*L?o8#m=pNdjQ5f^0^WlpJ$H`9Ynt6B;<7&-`htBP6}S1L8;DrNU4W_V^5b zk%fe^$r@#NIXgbRf@+b)koHg07vnpSu$ID0jdA0)r^>;+5=*Wdnw(p*Kg<3{-- zCj*N^K$ce;{|UPak8)M<9i)tB2e|!wDY@u4dM59~^V^$-6-TZIXHe+K571%!b?%}c zDfIOfEvLQyc2WJ;%fwAwVq2`wgcjG`!j4gNd566-SO>@o6<^N%WmnJB&2)<|Ukpx}Y34S;SUsQi2xW>xZ)&q1 zFELU8i;8CElS=q0Z62+VD1PPzppzt|I?OnVMo<&tTtnf%W%I+e%8MN%_WQo;BR+iM z_YoA_yk>s4Hm`bYUK0{lMs;H$2@~Gico0nSzIm*+3)~1?B!xzwztg%-&+G|K}p(ixq*^TXe}0QlPoVS6eDWx z_!Zthw|Kw{p&3OWluJFQyQ7fCZZb>CMg2^u(K+*Mp=FT@s^4!=ZZzb3xOsxsMq(|! zdj)eW9Ql4e#c6g`LNun5iA;5RI4W0*$Bcg8VkcNEGQ#BH?j9n-Oz~BWiX@MdGp-5| z2!%8wCX-*=oW!E;1#C`*Tj6p>dDaY%Gv7f^7O?6U7vbVCRwXPVEWcXsZTL!$=WLCV zABlAs)@U9is+AO)p@#W|Rq<3F`am+7VZjM^gR9suzFo(N8m3QpuQ7|X}iVtex#A5&A;1!j#6*rV)C&ViM+ z@w=?`YiZ|NO^TLa4wbFtm|JA$JMorIOlJ>6O|a)7 zdEn>RnG3pp_clor%BP1mJyc}nk{Oo4_FWX2!X3;rBRN<$M4()yrGThVQl4 z&ZbZcy7j?G!jz&HtKJ^OJMuaGf4kwCI-1VX6#M4$GtpI;2Th0n*$U6@`qx(YMAcuP zU^@v3j|pf(&E;M1h(FV6+&)I>=;l!G>QE96QrX+=o93(?ztk$FqmlHB_C_sz!>+-h z{H`hXWF{6-yW5!-P3`dsFLE=)%?9uCzRzZvHfYQ&_@VGu7*x_f(cA z-hLye@F>gcYM%N8P!%7R}#T)P(1jri%C7asju9@4HTy zt|GG#Cx>rNFvjggDqrm)ebh&Rx3H9LV|}50loq&XiITQnUBy`B3oLFg<7=DJFc*%+ zhjPs>2u-V*=l=bBkLw2Pc{blPv(Q_u29EL|(r7H5=JLkgQSGtsYeh4u_l47SA=W_>`%7ZuY6$xKd zP1bTHt^(_!!+X6T&#p*#ixnmsK#$*+w8!(laKUarle=h(#i~UBSK)^OGc?Cht@T}tcXkS^!^_#ZbM@GQY3ch` z5ZEc^pat|o(5#VE#YF`!2~B2Ii!zD6Gkj?Y`Pbw=GL+IrocvU+3d^m0XwaBhcYnRz zOk;*xB2RswixiyFaCq&2BC)!U>k#)tr@*dz=Fz~a#NvB-PG~k_$Bvw+_ANxGY?tAd z?{!@3X65b~##RnV-|jz}97{sJ_K9*a?w_!k$~(j$BVHHrA{7Qi)6a@+OVCI{`ELA8K3~B!yl) zBz=kUnso%RY5Z^M7UC=vCz^$auVo+C&lLvu)%u+jl4Ovc%eL%V%q4diqs*_wKXYw9 zb4D=wF>9RkqV20-XiIC%i|%W901AfTp=+E9iNW8fC+`eJ=N0VK-Mv%OmPM=qgCpp!95MA0v-jxlQp2G zMP=jX1s#}>g(sjW8rs0vFAmy`PjeS94Q_lN7 z{&XD5{L(mMnu)5&oG6*M<=%*M;=6R574iMv0U<|4W}%YhkQ``k^Xq#rudRPylNn{y zyiagzD=O)Zdk-8@ef)$d6boNE4Q!NDyZ9`VM;QlIGNce;9Wsq4s62$6qT-_0H$O&| znwPZ~l0c2*h*W3{5#eL+tT~7?6k7~$&&x>!7EBj3UkV<65e=vmg`bP)HOg=e&Dm)V z`pn)FCWZI#_zs!|)+O{lO9T#yi(uO#5s;K{n;og-B1z|XI@u7l9=PC!BGLEOY`Cmy zbGcO8)^KIpYcbC3Nn)en$jWni(8?th@EAc?!ur}Q;cCvWqK@8^6N-#V3t<;m_de+v zr*y-fni>A4xrjlU^#6mZcL0(l*rG+pwt2?3ZQHhO`;0wjY}>YN+qP}|x%a;RD3)+(2XW9H(-0AnhaEsfQki``}-1#%5U|GQ3hVY@WtAFrous3P%9 z2EuK|vL6mnzhqB6(Ohp_g(l<3A^L-DM+#YJBi2$#>-AT|sA-{z6C$dc3t;i!`Zi2A zj2p^-zMBM>6 zW4lOTI1x_ybAkQg79s!BgK#1M;g0^ekCGE1*y&~VyVvGtTJT};9kjnxb~kqH6+KNy zD0){{N_Q_;b7WLaIxrBOzMa0^)!DVkV$Reprjh3KK%}pk%1W{YUDhfLka4`GrkxSo zxQ=>$w8-fh@9HX$PIz7H3R2n6!GOCAon`130A-$1KWB;unM`a+J1LGp!E&+0y#Qv? zjwcbSuC4~GD;)5bgVS%^sh1-M1b2rq5U?Ckt0m40ayD0vlG?h0%kz;OtYH)#qbd0G zyryAxs6$CO;?fvt|H!Fb?@sJt{LzRN{4b%H+MMvhWAP1MIepdaM*>P|YDKO>jIolU z)YK`)%Z9m({vEDk{^BXnldP))Ct^NV9 zFo5*g`TbdXYSM)tV?RBK+NiDkR`KDtOI#bU7o9U2p$!+0-26J`9k>0 zKs}V-p^>*1^cs_y`-(4;8*0ebwr>?$;2E(G1-qUN+R95iAc&T@(I=YSijJA0Vu6jx z$JqAs6a>F;v*F#1#%j{JDxPPfRNELx{;1EhjMr@(VJ7E@5~-A68yb`97c-?|WG)g} zXCP#Q7IMP#94L~pRGuxz)UjfCefpr)_0X+3?!M_Tr&@+wwZS1=hx%3%d7WRRud3>J zpZ8SfLbrsjAydattwDA22`?|LznuMKF(u-#7n7N>=tGIYPZ6vXGRQ!{*36l6A-;Z=a=GR8@=~ zr^K>3QA3ejbWE~20uXw!AjG(U5V}~HC78UyJQ8)!|{D^v#%DdtW8jjJX5-fg_JnhX8Io>ZiK(lHu zYg^N0(7YIRC~KYeqdF4A%l&~&xlVP=tAsdUv)#9!&K^Dqn4^p+<+tRcu`j+*zt|L8 zVIFR3bw@RIZP~M(4x00hT9O9!bd*Y#gIjr1q#q_b8A3)@+}sstf-A|`=AQHOLh`-s zP~K%PVKkYyE#=|bT5dm%7~|^uCIk;v8r@0 zRI;tq+FIvT8YXKWNz){kaV&K*>Ob`UGN)KxUPm@^PKQJrc3QC~aS7*kQFWR7LOZAD z4A%Um(ZeEXf^t%wwC+JSWinacN%+egw6b=chzukJ7K2PgATtl>_yBXA!l<+q%$hejb;593|eCKteE!meYzcfCE%=W+^__4NYphibDP~}t8LATqr z1}yP5##z=aW0l2QSc~Rg;((|4O<+>#L0V2Sf$eJp%`=8@4Y3=lO(TYIf@_T-T5TguU7NKX1J(yFd_(Tw!wxov zcKUeD8Ff_YVV0G5XyckR^=Z^!3?iq zS%1niqY00x)zz&P#;$Eok}j9)U+^>tI(83TOHIuH%GWf^)f%)wSGi5D!04-?wu6(K zg!m@M5TX?$!EwW@7$qjGT~H@tQ!wHC7Pk+edEO!ftqDt}4hnh-Z=|57`mVazGEIt| zw*W*>l<89-&aQ2xv({o#rXafb<&tuzl387^T6gd-))KdeeOmBkk+bGNDO3qR%iLKj zoz;`rrG3_vWxyQV%4kCU&-?NBHIpFZk-sYl9=_SbSoj5I6O=0;9A7E-LJa9|txKtR z&bOC&udT06x|dnqnt07I2X%r63HuH9y8m8Dfjot_LKkZ3@ z!h#h3v7f^4N@#|r@i_(J=Yk@;;!Q+B5UtnSAQV2Q07c=s{k~PeMTqF-K;R4*XD5*3 zjT&d4^UuBtsUe@36giwcU4*v%S~kqkAH}I&2c^T#URU+HIO^ad>t@COb!q ziJo_l3)$|~7&`~x9I)MKv|9$T+3GvYpSr}So9@$MdudYuCIALUV7vtgg82j>0EWQi z9|aHu!v!D+WceH0hs8g-tA@-7(Q|?5F;id?5u66gRUNTg7B=hRMl@=C-oC+LZ#NLx zu<>}mwRW4obuBwWcOoCwI#8GMfgR_@)?d%7V0L^m+Xb6ojJfLx)73C1-S{XMaML~P zS#HuB2iaOt=Kwzt`P}`&4CH3N%JR3i1Cwj)2HxCONxa=BFylDp=FD+Hh)b6~J-#rWTC1@<4pe~bYA zM09)%(7sYI|Nj`u5Zs6QDF3fF=N>;9&S?;EmOWVZ8wA&0rI;{ayw zsUbtRD|=TV{OzGmS!8gi%_lf0s7(QXE^_-7X!%`QjND!w41RyEnFIMXj-z-0dloMtvM5O?3ss@}iC)aIgL!ht8S40rADn2h`*o*A~OUfW596U?x*!=FfD}5Jzf8&P=~gW|3w{M^n^LmB`jDw z{V8nA#wMA4WVyH>wbabUQZWlE#rWC10#f?XNu?xjd$7dDT)v9G={yTgzKZ#;abJzM`2uw6bG-Y$6Ke97n!R_b!J3Teu1;vg~2BRAHwsiFvx}emgZHO(s`*LX^J1 z;48jjQ-+CkKtrzT({MsHbl_=ouP)kSx|6pJJ-z>Q zd$_#Z0%R8!!OAHGid9Up*Zv{Wp1lN71@9tk%@e}PL{^%=KNaJg28^;W-c9p5)WKaX zl!2ZjZB2zok0H8oC*|MNWvb>jw$&m)yh6&3$X0&sn~e-~~sW48+EDv~f?QrdA+^7O0< zT;6%M$zc(3QB&+16x?L^W1qb~`8C|-z94y_g?tOhH5TERGO#=AD&fd?0;0eRgFh~i z2o4Z-FHj1yXFz5VZPt$NHpzsmxtrF@}m3o~fJ~gZZR82Ke03|4s!mtnjijb;Aa;t-QTIQrr zuoPUA5Ddyqh+@yDW1m=Rt_JZm14iy(#0dcD`xD%o41^z0XHB&4BmFbb@W4}hPtW@2 z7m9bu72^V~%N}jAu}94nrSqZTLZ7jnEF!0oT-PBLuUO5h0kZQ|{ADWCU_%?Zm(5FS zQEh^$DXo9{@GH~m)*343c3Mt9)U78Rb-PvOnHQtLlK;zPT234_zJAKU9NX8HMtZ=! zi>4stEGXmp_j1sReq)tDAG0ZvzR}T`N(AzY*Wu+2nKVT6!tAiNpLQ|NHrAseG)Xc@&2pOBCJOh)G6`lR#UjXKFDiLBwqe1KD2wm{s zDWHrefmT2i2F=|%t^3um8O$TL!K&c*MCTNb)uK{x7}tafxAWr=8-ZPYWPJNwPzW6U z-_iAeu|(W?aTDA9*NRW6-v$R};2&sv zj+k@Q)!>cJUty>F|0v|~8c?xKcD(C@$?x}wCqwQWg~KZTLiJtrq1s$gegHD>uv< z0400rE-wHo&MI~78UX+*_uu%?(triQ*uwq8^W@|ms~eX>L0{2e+I1rgODTqxPg41c z7>x?8`c2BSLN2XA(!Pvyv#z1lbc8B-AJr%9JpTu@VWtDRzm~M8JFo=Yj8Saw9{cokFq{_~x81{gc#}F% zT%Um)boQ@-At0eUKsevaYR~qsQrtw`Rfmop1%7+Fc)Txw6H~va{s8TT%v#G{N}D*x zHP0~@0sFx;pwMbfa5$=LfszW+0CJN-6uv@{cohnPnh530aiO3R#T%L@k@Spr=fION z$;91X*!LJcS+tz~UPyMiol^Mms<)HiXl$`g5=HW0QaUT$Z<>oeuHB{NI>3DVr~2Fn zhmc#iC=;1Q!OfjwIfT1+p)N59o$M4(?w_CU`=i}R!OqWD%S`f2ceP)))n_KFZ#J9n zG&W!4@y$>}O^%OiPBaN)=B6#pM)l$b{^~mb$cF0!SU}ZN5)TAx+=Rcke+&*_mXjUy zkzQ9@x4Cxbs>sxvlg=OEVR`_WU|N1HNX^OFp^JrwmX1}Qi0sH*sG7jXPT!Z4hnDui z#9|mpaREpQa)Kowa)Kfc8Rw;L7ukpV`^Tx%skM||!@olSkrsfp7*mGb(3IUcgT`$>q?Ga@8`Ts-e2$KYZEq6+l);bA?+zxuS{ATW!5jS z%BY3nul0$-Eon63gf3!gV+dp%3EjV1cX1t^-39j%gd^JD!QrdxT~!+M;*0axCEJM( z#Nu)99?WA5Q?9DY4%fKD!}94gT8kIpKzqzb!eK%uSeZeEQ~J)&&yvt?Bqa3XrNlU9 zr=R_M4_1gDZ4c40>s~K8Xu(AcWCA4Co7AUUddhD>_Dnp-=sp(CtINxNVIZw&c~-B>@4)#^9EYG< zWs-YM%~#GaF74G?yW4Zr3!AAZO2r|eb28DN!>;D4nUM{W&y;&Er;w-oYJ+z|C5_zlu6p;3Ip zZ$X=>TWl_dVbjbq%yP_-F!7u?Omjn9GyV6Q^}(%L5AqQ&^yES0>ad=0pq-i6wTV4MY~kYMxhs%M z&e~})+tc=?W3iuzs7MTx7FiTEA-I4WuL{!^xKEgL&8R<)ex2%R|C5;$z{OHIKs334y@T54!+!g5m)Sy6;?R!HR!L z_DaqK!!ZS9qrv;wQd72S+AgW?yC=;Bi=4zy%Y(1iVg%;;5gA48u(M(%r_q?3Y_kJU zPwzASLlOAoo88I&!R4W;kPuBUyC&K?4)vVxZrcBhual0Vh+AMdXeel#Y#`#5Srpos z+|EfuXym3N-ixG_-ALzp!AM8&5EJ(|eYXiJVk~H1^uN9=_cE4!6MNK5E6w(2#${Xn zk!-5$MN$F%QWL2^I)@k7syLUqRa50S&d*q{NRYMBiICaY={%}mYA1HQ@Nkj0k28-E zLtb!6=~PpSVmjb2&i@K|P;$1ZeE>?rA&VzHoIQMOyE;vb#d3>ohP` zyt*ZIIa;R*FH#Jy31jBWcW^eCJG<4Xt*i%@oQqDQD@yoUITw3s;oCl~_L_Q|<;7$A zE4x;(7(_&vP+;nLs}QYW~b0_c+O`Ei(`B%rGShr?1*K((Z2MgX|if7qU^*5&SN*6=Uh5W6fdH(a1i`m0EX@e;_)WB#BNkUp^q?q$B#X z^q#UsjGOaOH_KJBQ?t3~_N#C9-tVeXI4pSHyP9wGALv9uw2=v{W=!?BjZ))MBE77c z^4G7RH`|h}th$8S$*9MW)dx2E*>U*Q);=399)ryihF*KtHrAMokzR#{(gnYjP+f}S z#q((z%9eBvlt5%!TtO3;4z#2RzMF(=l%wV>j+xL!vDbA&Ki8L@~{KBkRiiBxcjZqL$pW%zZzt7Y|{u(l2y z1t#>4yoII93#Hg{RF!fhRi_w|G#AYyjyrD+CTbwr#^uf)=t85i>QWVXNu%Ktfa_GvdIwT&Dq}vDBA*!Ax9|J`;a4+%7iHG8@ zQPpd5y`q#cWg-SinSiFlHm6(3%*3mlPTR`px^#G`}zf(rbf*r%wYL>GD_Qwr4f$V<1%rk$lVr(n?&iB*(EgA*>{P{ zprn#>Flo?i5bPU=c%&l|OgiYbsdqt(Y%39f&QBw7h|*oSRL}+d7N$wiTJ}drlN2II z{;*7+$K9>OASIu)ii@pg$0&47#qb`fD9jqN_CF~#V(i7GCQx5E>3DH6fw(vN=wMLO z7=(VS_zc5}X4wUgdc`YCDlct{{VU_koSXVHThX$T;@9Qw_Wis*0qd#Y)hZ0ZcDf_c zrQJUR>lyd#5RPCw+e7!>CXB##TGAXU7337CyO^O(7=!&NCmJXv>=39Mg@>1dKSUTN z)Y+V!G^|i|vTaYkj1fhjK$3pkuM)Z=Q1OO7{f} z87Uz~YRDGYtn>I!=SxHm1+gIIm(bSb>E-i1^;R*FW&CnZb;y9OZS>k~#!gkO>zMEC z7Dscj*>oK(;Zu~R`JsZ1JPu^9&?;L9xOT$;RJfC~7_=3E$n{JU91dP3cXx?E@h7V9 z+)Sv}U0#$jPqouMmUGW(`L>(KWB-g3)#e^fBk_k<#>sa7z@=qnL>y zXOc`ESD`L=G`R8T+cOZx5{qM2X^Q!~j)(P1veuZ_@p)GKFSfGIqZ)(L)_7ppLa1Z{ z)$RiCNK+yorn)?oeLhC3b!||BFM+a>59uKi=Q1rXxLmp{K|L*fIb_Ev4^uZvoZ^|f zxt)KAk1`L(Q7|CK*t6pX-huPNE>2+2(c&^*eVXAM8O)JICX!#mT*9v3jZ}#% zvFOg>i?Oz9w?EeT=`l$Nl}Q}aBN0-9NrQgL zXllJ1-emcA0-^KA_dn_+QVGh*Miw(VyN^G7&)>luF&nGcp*>KN5b%K4H! zw0|U;HR_0y$+>1IHpqsj_1zrUp{2nv6f@Lr+yn|A^}LQ zzuP~(i}^mEdCl3{-s2^AW)k!CfEvg+kWg|#)Tu5#p(lOk?z&#q zoOZOhin)g96qT+eFC)(qGf#3XKnc9{uO4N2BR21&?m~t`zRLiVDq}COpP9L>h_frv)k@}c^#(G9EE*DzGggO~9G@DS^%#*lQ9XD(Y8DRxOm-mFBnz^> zh_N#>MT`|dCi+)dHh2>yd`t*Gg*T^D>z{ehm`nomAH&Q(+pv)doww1@ZvaJdycp~e zR@2cz1V?dmR+kP}e@Hq1@z?4gPo&eS-SPuv-dN))HB($F*9l50Mue!e)xkr_FqaWy z-+BThA=2mD27H-xWEF`B2BPRn%H=4xmVLIU&H(cFXpz1o1NXhw73#K^JTWyJ6`ipN z^a9biZ%Y9|0HD1hmQGf(8<5%zX!1%2+3k9*<;;YMBdxL;`EzWu8G_d1e$`|3c=T9D zXe7|T&exk_C#JSS{RPhcGn)=9QS zL3XT_%n>S0O~IDt%2_R#v%aP`IcmFhytiE64o7Zf4sEZDTC_TDxtvo|1` zg>yoYf~*L1MI`!+mvUR-{uP9Q6$r}y?^~M6^{onpsOf5~Erp!~!JCVymwKz)cj2wS zSgs$AhYw@Z?#k@%>JP|ogpL#^5xnO()C8oD&hJsE84CHSLeS&e1p7aV#Y+^bhr;Z` z=45*NUbOUU@dT*o_9!-CqRC6q(CCgUHn5w$s)o>nI!@3#p+1a0)`g3QkB4VtYuWx} z6G3vhx{URt=(fjX{C6_2X;2ytih`LB`lTnnOgLp0!5M(qBaX6(#O-;!Q7GR~1WPiJ zQB0BVXJaM5kD76I5WOyBpQF!mx(7tgeEVl(A_X$h6f2p-R=f^`7wF7`5HHpgJwQ;y z=V*h-prL#e&mEW*#2qa!Lg_n^U5#1wh?TV`7_r8uPULjbjQV_qBPQJc2c)`z<6kbh_p=&@Dh9r~`pedO}YRwvMOz+;2{^nVgQX1Ua&c(o4 zfyQh4(3(|#B3@h9pgp}k4!XUxnYUVnK_)wOhDXBa{_1aIbx#P&#kiT%_%?Xj2I=dt z^Kev~-b!=i7Xn7dvPU;ZgBi|sw*ku^8L5}vP)oa-{ZpUK_TR!kd#$t>f@!Isd226V z8cpn(SH^KPNy2A>@#r^F@pa2O5xK`)>*`nY-&;5ecLAzzc-8pkZ=x0bR?{xpwgD2J<3GsQ^rtg9`o}5FC zOh+?7Wisg4Hu#Z6q)jtE)G*9YP7SLnk#*)QZlnr_IM*zMliW?<>CJRT;5xC`XUG=Y6}CovxtW(o7W} z`evxzDUHoMyRV>?{>z?fLU78SHVk&6n@6Im$!6YL-)R#{%pnL;dTW-khz)N)C z$Io=6GOz1g26rh1%ykRC;a;p4b?pW#n#XEP9BuvRqM7i|q*=Q@r`1}D&66ZW9qo_H z#F(C-NaCyu%376KTXOV8*#S9uRSJ6GzRRqan^BoI`pPFB=1xq+q_?2O(9GLl+VbL8 zCB`Fsh$S2HO{TGH!9ItY1bb8(*_qA@M(pq9d7wlrR;^JxoCs0s<6Hu0)|t|;|fcdmb?$)d2U(beTDXAax~0!7aIEW+ktO2RUmfjqVA zBWw`~S`UG!F{3%Xj8hUcWM64$Q!Cd_kpf%%=M2B-UoPjwmYNFwou3wPzL&=Emj8FS zZXt%ZG$+=?*8X1;YGY2!ZR#^}CstTd7!jcbhZ%1=tC^mf zQlEZVaeTGc1*rN_42Sfci5frlq+i<485ZMyFihXV$iBskDHh%EG7lUtw&Tt{M#*wg zzl5ZZUgDg>bffy4jDrQHbk>!d^nItn!qhU}9he*onoRqPqcn#4)q0mWKXo?gDic&5 zqppXi5a@0THq#DDr^MH)mli5IAt0s*seb(tW?}a+d>^7CR4Nr(yqT+S?q)^*>^r+0GBXyE>B@A@oKWAV?{kx0p)z~w#7HfTJ{WoLQSISS z{ZbA?-JNO^kkam8Q_!yA%u!i(N!xpZJqna4BoIgRaA`%5D|}L!QDm(y-vTU)VUh}vQF)W{o}4Dy)4S%&am2j&n?>H;7x9p449vt{P@Tc>~dzSs6~B` z6Klo+Z1So#yEJWjF<+VSY7H31 z0kSvt@6pYM%z_lzX93eOhPUsL*_#w1$D`Lj4L1{zyd!7dqUtw|)DC=lM#YD>7`($D zALu1-Xg6sY5B$7CBe!IKHdEN=a2WeS-N}FU3QWMJ?=g>m6&zoooW6oMc>4$5!5d0A z#Rezp_hlM(W$L$Oo`K^*h!Zc-1R^!BNFlNh4bVEkH7@QyeU*HCPiLgyyvloS)W~`q z`dCVqs8r{!T-MK;5+rSBx`I6|@Nhee@3iLDpsTcIFmbPfp{kKCO>9l%;?q>#@kxao z?T;`BW|YtmtUSaH6`2HN+w3nK&>$AFh6uVPu98{pc>RtczUq&G=nBH7dPjtq|PiFCYKFY|$VOLhuKFOzY+zje*ntT|>xlEPNvf$|sA_nrJ zcoI+tVM>ZtP#0G2h(nL~>f96dQa{mA?0k6z@FBY1ey`e&_7$4L7*X)&Gg_jaIZz|W z$}kR8UJxiC|705#8-<@s&>912Fm5D11m2iZa1kiam2g@YqPvhV#YUFyz;+V+a&l7U zye@IlAvIG;7%{oEXrM{aRm9xRM43MHrQ}MBqlaN7$AtNly9X!7Ddu##JE)pM^}fFp zzEM7v8V09ms0H~|3t!AHORCM$7blz8;vJrKot!h&7f4%wv-vdu$JB=;q$flMy5_MZ*QFeP*D8P?-8uQG}rncYd3^T1d$-+i&gErWH9HJ+LsCw-S#In{aOhVsZ)yqXGa+^MN)GkDTi7CwQddU z>qg@hwpgPoOHC7s&@w+>S$D@*h;4%n6#((ku~I0`Z($RvcvEA7P+PhF2Ib-H;MH#d zrt-EBg9ktOd5TbYR18^{6p*|EkCxR)LWcFK+130!PYwBXeVckubQKSAPQp_`fMx@A z8p_yV<`trT-c&t>W_hL~z9_lIY!x1YikOq3`uej`FY{A^rCq!bQ*+QN?V{K!WP@wK z_5Q2b?)p{zG2P;yWHLnBb$bN@@DdAOcXW4hBxrs``a@DJK9po?PoXuD=QPIS!5aF5 z?N&2aZygnOM#R|R;;aS_mFKMhicF9SRya()#a7?Q1dxKbDt&jX+MbtQZ$L)8|n-)<{I{ygj8S<<{}L=-sgddPSL z*T6Nmy`pI*Vv)-GghaBkr*fxGq1;U8^=JS}i#AsCGYygtk7Bt{+X3uNAx)-;2*nRrDS) zWJFhkSrBMZM|6!i1>oLa^#&HXRzvunwt3%~?QtF25i=h#4LK|kNjEX@GB!(%={rw0 zTq>?+V_yk*oVG8T(X|bHgON03MdFG>8tc7eF#o@iRe_7d$rKNVi_weqvEhnj&z3%M zjSEk>-pQ;{9m7>-=;s@1Y9}EW$_}l*Y0pJL`L_x1bTvq*^9`aoO8GTzd{KJy@HO%E zMjVXuu*w!~1{-tG)HPmAJ)>)Q!_&5>wK};5jfddE*<4YnCD@2E&mc`9HitA+3X)EV zspM8;(awnb^I~Pgcjsc-?}#bBgGG1_XaD!G26n}Hu;|~*ZI~3_{`|}Of4xTj_qO5s z-!B}@;5eKy2dXRW4d}}f(S_ghx78o)+h~1A5 zismc%gtF!xb4>$Wa|PARQnp$r-eB|g9F*exIn#PolyxHrAY%&yJH0 zYrU0@npWpRhkWSdol^^{RI8R-W{mcWx1wEZS1SjmUax&>ImTdnp@-M ztzneRZrO~`LRVr&`%>iywSBRC)Gp--058sss(9TY71apuP=q5BU$TaL^c8`AOMX80 zwb4b#mtOw^0{%S5H=y=mhwF9bwnI38+{9*ANYD&ow{f8X!Y`$gT>!lsJpd$6r7I>v zfc~$VmH=C){e+K#O1(Hb=q0q^b;m2sg8x+UlCwe`M;iz+V;rIXfhPTJjiynP&IY+I z{s0fB`&XYb(R~_ZLtl}85agEiRJr!p{W`Vu%~?&{8q$<3x)l(YNipUuG+?g6 zk=iT7FZvpnQw}-bqR(YR4A)o66$?9VC7iZ!>486BYQnaytnCSiBmt4cwvrojESGBm z2=L`~r42Qj>t*B)ULj2TYfzisN^0^bnA4+^0z?8s86Swk5@_VA4gWxmimnc8UP_PE z>E|U+uT7zJ>*AXo)7j07;j$$U-T#uHlk^;!NLA3Q&w=hw30b%bwcvMZ0=gs?uGwhc z(!bwrrVYmZHL1!%yciR45o@7DvU@JF?gi=)N*x`>oD(YRsNwxbl`oXAG;$7R(w~vt z&nLMU-54m6x&&U(=~^!8cV*57I48kk_dN1%*byR_nUA(8P5l9}S-W1aCoHl?sB!kK zs;#&FOy>%bBKNO%{v~@A<$SG*)||XY9na-?M4VfH*gzoszj-jiUucR4T+NLw_-*1N zoRX*IR%;jN*0rbID=~>4EvyTKA9DSg{O5fu!5e#E64u)J*tlsRgL`RytA&N1 z;i4d9D5>5j^6>EA(D$D(&#Wqg3;(c5`1k^ee(v|r?N)w#zFH#H>}Yr5TNIhpEElA( zgIjKfjR_Yzjr3`ka4uE<&Y>4|Dk-ReIU+USQ^$|x=CTCGheLwe!C(C8hCo-+;HAOV zFbc(eC7>4(6??NRvCoZ63TV=M=4-Yr1>iQ{v|Q1X(+YLSR*z|I$UJ*>9Q3wtvvlU5 zk^RW0)>zBPNt`(SBfvn1`_>C6QN69B_uWIGN9wBdXBs~Z&qCkzxX{DL&IujI?0HXM z{@Zj6{J7grgfCD_EYR9`?D&upI%vs{38fZWbn-&bNnQiUg9tFqKM}Y~TA1KLb6zsq zJ`HCgsO$QA!sOn)yNzpSRQ0gH@e8}qSkRJ+BfUl-xmXkpq)I-#&1nyXXS)&s{z#oy z%~r5xQw&7e{GecrNsr8Ll{wB3N@yN}X+8x}&c`(i(I@y_WF zAA%PG9h|Rl!OCxX250Zopo8S>q~wT8!DdIQtmpBKx!!GBPl0gDVIUw?*#8;En(J78 zsH8^fXpabCQaKv0CC3IxX{5?xDEPfR&gKzp9l5r<-P`0jJdR(00I7XV@Hn#zC5>W- z5^8)W9l73#lrs~@L#QQ4Rq}YWaCxeuB0}S9=Cd=_Gpp(q(duWt#N$sc1D8DvdV;vs zZP_v`mHL=)Tq59Cau{^rLd%2!U}$aCwxj`^>GN>Av$&GpCm(y_E^!N%9Q7O#UE)IxF7mv_7`eN*iQ>B-6?a!`!wdZ&Ko8Mp>TkSyForhXo(<*RJkjd%>~Ch3@m?<}2?P;5fA^#RxgB(FeIBPA zfjvj~i;$9&DAMTD`sBuAKr0mczI^(zDIT;Eb3>@<8Oyu1wi1;4pTIm;p2l!l`4a?hmoqUcyo4ivgos zZg_)w-6)&Gu!L?W>4uMTcEdi01(1(m(Tk!Vpif9R6!l(!g2y@D;O1ArrT0PK1Q7T< zjFamQxwtr*loC)J$a=&umoQIF9U`scOD_(u{oho3S((4GF;P2zwBxPmQ~0cShk_zW zFVgF+?|I9|_3l*2#L$md&1qFBmDSAq@$fXSLLX6EsC-P4ebRUxV1pbs9efO>&BWV- zZwk)7FVPK6gv*4qLo1^5>9Cn#rYu}^Mivqhx*pr@s_EWq0-KEvv@t$q&D-`Qi}cZp zx9Wy8$FUu;Xrh^vyo<8er+Ar&#L0$_^Y3#>XPtwe^EhV5`u_o;0a=L`whb*Soyq|( zN7GCB)Ml9amCe^RjU9Pfp)O`|cZ~VNPh*b%RvKv|0#y4J8xQmK+-~CH2}BbCr^Bud z79FpF&7MffV(UlZc_Cqm(N#=LV}NSeswP0XkE`#)?2yV1LX#IfBpOE|tX@pi5dl-mGP5gW+Mr8#Dr%XJTW4bu0!T2i`g8`5G0^yl0YA zu*w=OnA{A;ccldj9u5z{hRr_m+r&0WxXhzuW9({Pw{iv6x#!s=Psj;<3S7qNvxFQ* zL0$o%@jbMu@Gm1{TV*Xbs8x#s*6=ci+X`LE|&Pap4(@j2t6jz|5ZXkio*P3i4f zdXqCLI`nP}fO>(`LWu38NkLfFKB$%h0B216GuyBP0+xNuJpPLDfSMHwD`aq#)F1~R zmBfe{plq%6D6S3b@o>QqR%eZJY`snHaiD4A=5%85I>Zr7LDKLG7r$17QBu~&QwUDQ zW4dsH_uyDa;i`)yw*F)FSBuD<|}tKZpD+=|Pnms@p{lV79e*ZB1xT(29St{acB z8KLC*ZNGOCzjI;VrU_zr6_=MS{|nRg8v)s(hv)A2|3vxUvHt%(3^(!+OOiM9h>mPI z0`&I{q3&3|4lR+_AiePG`5*PX383Z+A;Wi^`naeX9)NYR(e+sEQIZvU&~O$IkFRT~ z9;IOxzmx%D&K?FBqQ|o=3^&*q>3ezokgSXRc*^X0K(c&wC4y9??pCnYwEzYIF1?naC&hidQOtLx%va!qTtEdDsH%%+nAX@)0=%O=dB;q?wDkK@cSr-648OiEO*TQMYrI@$A{tl1OBD|5TU6J6fKCyurkTqnkzuA1& z2VxcZRDKB4UJ%!^&-E+}=4Al|?2<^V5&A8rb`1fTb$_4b3V(LH;C1cAx(y`fP&*>j z*GCf7=$qXl0XK@ylLp2|hdmwhJ?pQ*J`RqWIzp_b^!*9s)Qp4MR-^lQ&Wt)ovBf%x zEUzIHL`(C}Oy}&IiKXCfsR6jNjNVZqj=_u6fxlWTgR#YeELH)}i%~(BvQQxRth^;u z0SfDZo_qK{7X0FemN%;lP@-o`i;9zS1|!0gG>G(BV)@!rII*|@ium@CZ$BS0MD(29 zHS0G@`r4-`+tT3|ecRpYcl~M7bEvkDpa`uzLf^K*T-7-RYm;5H@|Iqu8X63SHHV%L zoWW+4tVMCpaIcN(oU0?}4d<$wW^Zd9R1Y85=qyxd%pD#Ce2LV~v)s3s>(5@g@cG7l6G7w;;Am0T;z}PAcstzPXw zOy$M}MA{+X_(VI6CUN>8dHCOkapa}!f(R8wqvO+ABB%z&dje5I`pv@)TuD1Z;mJy{ zOj<~EfdMFl9;P3YsIR2#%^9Z7|HdYV4$PSLDZxFtZKb}Ot$z;TS$b2} zFNX4s&-#axVNTaxN&bXfL7ugg%^}o&v~_~2o@Z}#qME`|8#vzYAL5^2f^UqbSI4`6 zx(;`KSYjwz1@XP&PVR&Qt|Ws|b$|fG#slFx*%Iog{@s zI=@$;Os*(O%U`K&fCy2eAHIFQHRuI23EH`;Uuex}-fiL(>d~g$1a)`yMPnD9AJMLn zeU6d)S5hq{rM@U3ghwf34rXV=n#@jP*mBmYHARAF`*I(*rgaS9*>4U0z4`!-!pFH! zAUj0K;qKF~072k&YP-7C^Q%OL>~ak_vb1#eNYpoUibFAae_43d{zTVW&;r}j@64BB zOqP^#j5Z>QR7EDW^&lDwfs_IGf7rXH?$ElZOVqJ#+ctM>+je$r+qP}nwr$(Cvy)Wn zt9DLnby@fC%sJ*WdI#$)Q&-P@c`e=5=>o^Mr@Da_(z<@rI)<_hE`;Dsfn*>g zmbx0zUjcV3z)n0LE?JhBq83Q^rm0Z9-xRtF#lf7|E$hvLFfqc?yPwzL(H8#^z77?F zOOnCA{fHa?C;T3kILB#`{9EvVTO4LimO5l9x91o3HCR4f|BS2CeC82di}G2fL2UkP z%^WxC06%}7L$=%V19q}kU>H3z5dj;dDbF#+i>z8i#H7E@&0eVwsuaN{Z{a%(Vi`4{ zH#TAg@W{h@cLku)vvr>2h{M_{yVgH|a5CZZ{DM{gP0Y;JCM>m%a>bei;-=9o*?1C7 z13nbCbLGwXmN1=F{h{OX-Uu$L#-wTPb1Cyz9E!^N>GEB@DbV?xrK-kFAoS(1H9|le zD6Qe7l2d_I1WEa+^O)E4y3+R^Zw|2HMT2z<-Z5Y;FP4&w0ptCt;6NGn9I3M+jSDiD zyjwBPlRoxB-ZNg7Keoz5rGsk799DzDdo+z8Rn1NyO-|@&q)x#~hMR0?_=hM4MCP*R zPp6na0HZQi=n8$Y{(Ft_{a)?%BHpzG`a?>8+eAb75}MRmJ|?=>1=sIBC#tyNP}4JP0c z6HR(a7nVMibQ5s1rRmYQ&)990KIED|{!YWDKWbL|{|JM+h}CY@n+#M~Y$li&%O1!~ z(|FaL4*W811Yjs9oI$n023&@e476?x9Ol6|h?dP2KaMjwH4tiXFlr>T>Akh$uJK&Eq!lW zX(D&N{L2lAW9~Y({3I+rnRaXlxQ7grGrt_2r(EZlg=Pwb%FWl>FdmG}KB=&d1O+|q z?`-V20_b;xfWjgvr_vTcmVQ?;1U>As zWm1fiO9vO;7!s-xg)?w_{G3l<++*JB5zTZZ8HXBeIaxho+3WRIk<Ts;rjT{@D0Irk3ZXPR z>%kXAmsHI^?m_n**~sO5@c%uL>}+<#mq?+BK^IoGnZj0&CB{u;b+PS&vBzq)DHH%U zIn2{;wj+L;E2*jq^g!E?=i%Fg-Je`vQyw29-1 zTeaou`nj!iO||7CY44{v2T2{<2Rr2cn{)=c_Vv+qI=4@#Ki)t5I@y zJ!HpKqsu-uc2lwOLE|-u&5U$q5V{R+8IHsPka}VlXAVWiat~BrN+$4Bw;G1~$wQwA zvYxcZX;(}_VH%m+Kb%0osQ}TuDiu)H>GeU8Eo>S_34S{%DjHdKdm;%09mPHme|>_C zZJfQa1UT4opoLCuLozH=jl3T4n0T76un$}PGthFY!8AR5lczBZHTXFWyIElNRZN^F zi^fPhybCLpcL3&TUSzz;-o`3phEJ-o!?OraHr=m5@@SX{ug`MRT26K(-M}d1*ejz= zd>Yq2%B7)Z9IN&2Ez88TW_YOCuj|B#=wOMGAd##%HgpumPLqw{EjTk+4))~}xXqjm zpY0R{!8piEFZ({ALGSp|T-gnD+&lC619CgZ`9A)GI8j=0D<}K?FLm|j%U4d;=O3k6 ztm`dss+gxkEX0W`LW6M#&(T~EdKHed0?O}%TDRHW<*`}$<6+{bG3EGJ1#NrJJz2Vg zB{|;}=KxRap>yVMtmqZ%raoF6{WMq@5W#13C7rA)e_w9A^vB<*+%4W&3O9)o=HlYZ zE1nX7xH6Z2ZI@r9spO{}EB)+rHFUT*2Z$lu!PK;kYWHS`GNPpnOgpLYCkinOx@#IU zbO2$!g?S8A%J2VJr2mS@-p4)`*f_?tdwUGBHLT}!`Zjn>vx%anQPN`j-L}CWA-&8J zpX-vpo|j7bpYvzf9)n-3IS|<`Jx!*K5a*2}$rP zptOqShn2*hZwR|6KWNn$udq!`MTS~yx^#VWuV>?q-_kwYc!Q&pLBC#7yv5i45U@Zz+zMiLK?)Kts_=MEAa@)x*Yj$QgUrS#!QV~|9L2;5{nP>55$q$W zrd!&?kN3#X1bFKII7u(Kc7+OEn@fib#bRT?@&`u6@lxo9!O$H`59INTrRB0=?b2>4 z73H|qgB1~hWZI5eji;$M+ZS`bl+P6_q01(St~NVQkK-gVDA-98GjQBpZ+9TSS7DOi zjPA_d#^D#+f57kJ4|HdGtv3AENI<3ex8~-d_0=#*L#{~=AS1rDiGrYgHUoR@L zTaFN`5W|B@?h+uaosPgFsTwmNhpHBhZvX}ZuA3Ly|9QoSi17>Mq3~vNYQvuciOnE1 z`7+Mvvkxh2e5g5|!4m|>2)i0$GaC;`wFvGEU3Z|7 zn0jnkrfhwV>(A#eZ$x5dC%Oh0m|x;}$e&>aB1 zz@Q?Pug*uAdJB6YZxLV-=UBL`cy3u}etde>>rl(DYoV!9b%ko$S9Qq3TwRdwFS2Yc zXd$|yoP=sga9*jh>k*Mp?}KEvd**)||XVMNu#uB@w zPA;xbM^Z*sE+;4J8q=*o`mHFH<)09LwoplJ$HqXK+=SymVR`tVDTm@B6gx0m4Cut( zMM%ol&e@&X7|v4D7)N7SZjKihC%H7b*^z7nGVCHJIX=Um9bYSIEsESE_FB`wml5dN zYFc|n!Y%4neRU2|{}+24fb#WY!jE9GqZxXVEg)ov{}-PVpvNtln>T zwk8i7z#-R z8h@=Slr6!n7%eN6#Zy+wmVOyYkocB(5Px1CCtg-3k5w?smOd>t*6en6=*zf#y|C^{ zj5}J@BS*HVODxYBrgT-+<#BPi%eXSWP&`7RCsp#(Tr|*36wpkxY(5>=^!%O{7g(f; z5SU*KMP(%o?NdEowjqBl+sz@z6K_WTo2)dLExkG0E={U2k}PciS7B(y(w%4d_4`0{X8+V70bz7C?NW?}aq>#}Qea)>&65p&cF;`3Fqf}m^FVA11LY5DOj5m>+^{UZ}5FJa*?kLwS z__u+u$NrrnIa87fo+q~j_HrY9(pI4h6|ZvEq-h+U@sBo8Q`kd~WNQ$BWL zGi;1ycf5@)wBtbZG`cxAY&(H6U87QCNt4Qge2Zw2<2A7wRHUASW1DwL3KHAMVRDZR zD4qWw{{^_a+kf>53?$3>qws;;dP0I&S&0#LBHNzsdkwVhd-y*GCBO+NeC2hFe6tAu z4N(%7)dJ+9(-3J5LkJ?u{D)tqU>1AjFknHKF*;4#c?{?I73xsmq|P_RhSCfBp>oAMMvmRJkxOhT2jXYmxW=;lE!N_`KPh#$x}=14F!($HlS= zPbbbg+!3|Z^z~1JpCQLh=nufhr#r9C3W3iSXxli@FW=9-WC9U|?(LF1QynWz)-m6K zFL^ZqM=*ro2Bp%U;msCiO?uNy{*5vm%CYR5NqVQBe9p1tI^LE@zjB+tHxGZQa@3N#+1EMM(`RrUkTRBPp^i>M|^jw$fhIDhpac12vM@n>^2@Ecju2 z<#}q~&K*^ynq&UHemtrw8A?}CI(?k0-O=lYaic(u33jljrcMKlfKQ)%Lx5owo7yb@ z+&;sFo`&*>=8Vs_OWHaT8c?FZAcZ)hLrO1#*btD;h$CNZsR&UD0{***x-}*rd>P@w zx~4DM3s&;rFddQky5j3x5s=YOU9);^s5>^T)Hr_kLrc}}){!sSI3f<4Vm1vW6dbwi zrIb@lABPVj3hEkM70~BcXS)XAt7Xm5z0s9gKMrG?BYB)Y4(fQcT6C2$Ou+1w5PKLq z#WTt2ac+!noPv{h>kISc!%3tTbIbVIleOoVbz>X!EIWzx`p9Z32^YI>)Ya>2pJ98S z)6NNhq~rzTd(iJ4jyz3>>=hRJfX}OGx<`|1P}6G{P|L6?mv%_}r^=VXv%*ohi-vlK z1$u|%a@&}XK!RUz+y=3!Y}qDYPXf=h-ZQS5ZuXXa`84#q>!_~uS}CNhi_U5tti&DFvZj#8A+pgVV9YF?eR zC5O?yS3u$<(k`!K{_i+>7C0h?plv%O8o*QMtp8u;yrj>$Y!v`7$1SwEY@{wEg9baUPE3HaDL9lTnU&vG&;1cirsBcNqYJqT!3G z{lk=_^nB-y!Tx-fsVjkY<13H}*!8+E_tw0T?+jCOqeOZnB=R*X<&b|@;I3%xQJ=o} zQBR3zO{bX@R1`M0Y)N34?a@6=VBKOWqczAw5BHW5t765kIB9PkTbSc%KCMUAU9Zhu zRMt`>#F^+-7lS|sRY6fu5ibfVls7ke`4l|A$T27J^l>)y&*oXB2k+0n-RmjX)u-MS z|5H@TV)F>bmN1_`=>sQr{ujw8nV z4YD9-t`=6HXABcsDOPem2!=1$Vx9gRX2E&`yFntD@<~9$eoazj`Z4@5cscXCbo?Cs zRd8GV&KPdM(Qe(0%rrqNn>_8+N8X3wukS&53Ob*6g57%Vzb$K~Ugx3Y&cy7}wHlSC zX-Xn&e19+tt+|p#n)&LDj%+jv+fdS(7cxOVsTA3ip>?k1TLrDu{yE%nZ{y+Ket&z~ z8|l>Dh919DHlHO%%*cvN?@jF&FKl=b5*TgORf9TOo697n9R9VXg;`1l6vK29(pk4` zSVMYIXcV1`FsG!Be2jyQd?Us#!Je_f%%2Xp3}MsC?S3f*t+F1<#gsh@%xv=UOwnq}enGp9NU zyWEwh!kS?_N_tp@-IEVn+k(d(h#1F!L>#8_)4$3S&{z znB`z}mDxrp2F0ToHEVDx2W5roxxwouxMjW|Sdb=erwLGXvVOn7nkoM#Il7M1hfB;v4%|9A!Z8vUJg(mjKwb6y2XlBZ?1iLX&D8+W8DiYxLq-IosYAI z*H!UeyH4te_Z3^Y2BRxI!MLMVR-pmlPN7;Y&~li<+*}V9FUBZ9wofyv(_rLl3pjdm zB^ZNXC;p)R=Y0Fso9UuO6RY+CF6>ID`HdpB!OcP{Jo$V0N4F~#2zVl92X;{%r=CuX zGV^nHsn~ufr_FVe<8Q!6KZHf$>a1qUqdyPQGu7ImH>p%E+#hY?%mE$GMmhkuc#ou- zR%p(D(r6?Elauie_QKuh61yT3j`Pq0EZ`n3`Oi~f*@Iu*w%}Z?Sw?-u5r!Ux$s2jf;!7 zKMyCB@M^WbM`pP_=+`G;KhNh+o-ZGx(nMxvdjEWigkGWWt|z;8_qyi+ji=Vv=VJA@ zi^DGCrN&Y=FZNzAwVH!NV$*-rOMfa$Ul#zjRNS{LdyjNIZ$VPBw@tkv zJv*(Fv_g>L_A~eBpO@N+90r?P3S9e2M!U+iUy<6@nE~z0p-jK&axK!<=!5C!7sQEv zH=?ZuL9mESWyyf7eRiHVG&$Bs1L_xC46ZcaS#fK|Vqq6W)v$rQ-AyjuPiJ3bV7+mW zmQL&i*1gfMG6auM30C^Tx|3vRI-;o^413%H45Yb<#0OQaMX(z;U_*@KbFbXj%j7N= zPZmF}N&zcMv2|6pIQzDKY}(9O`Ik3DVfoDhkiV}Ixzo%pZ{l*S1AbKZcb9*Y^FB{m zN@B-V8!JO--$#Q|3|J`LGM4V#dw4xkjHg?6f0QB7F0QtiSyd~XIEQmvig;@biA z+^wTy{f*U=$Q9c$*Y#e0Y8@@HYre^!RfAG=jn!MX)av_8nU5 z+A)@^*C$WMKQwfX;MRkK&+*|n&o;-q=wu5R+yZqYeoSs~(sJny zCB%U{s{5z#tl`Yv5e;*yugR=x6!->!#O;o&P8s|QRu3x=T9VRTCbIb&^{I)5TS;i| z7)Dx59n1t}2a45kOpZgWk#F;d)%UBH&!8b6MeS-;4C0|%KFMe13Fc#~^X*xuijwK~kg(iL3I*ktEc1t3}t1A5@155dYw z(Sep7&%-=LOyjZ|PI5YS<2aJ_C=IagKAW)Xgtekt*|T5|hm9|kl41m879Yet=2Lse zU*WQHQVnlYf9LA~t0dZsx3yW|IU?1#pSWKw;;#eM8ZOZrPedVnr_=5oZZkesAj@}V zHT5`_R{G=<8dReL6@Yva0==_8d8TPWkY%q`k0yCTX*jEm{o7UFj3vYQ(Y zsup+8RmM~|mc?NRrJAhM+C-wPWV|B%)wvraHVvBcI{Fm2Zet7gF|%r*whbP}s>Vpk zM!NA;PJR17v?(j)m-?6>7cHsDrV`uac%aW_fzBL9b-o5ESro<+Gqso|UCN3@ z(J9ZA_^4G`ACB6<>U-4z%&|3`)X(Cwj%t92yxJD>qBK?qr^eO@)-+xqAlVnXJS|6g zoqy~r?1dZGMfOU7$Q9OWywYnfmz$GnN0^+TalBUXzqw%ptY_vSt++=o3y3+NmdRry zCT$2v?n&1_ZRkTfA9x)p-dIFFah|y$#@+3ns|qrpPj4Sy>0RfWf%hc{;~TZ@sWs*7 zrb-if{o&3}c*P#NaD1Lck-w@m-Eh}#-Y05BerHp0wRWF$O8tJ`K=Ly=QRwxcO}S2u z*_{tQQHJ7p5t&6f$$=)vqt(pRJ8k|m@R?J$0KQXrF=pL@qvy94C<_<^u#TRYq@MQ^z_KvZg|_VU^~24AwsX8L!mhZd$j8$2XSU0l{uzxA^^)OGv`Y-0gx ztqTt?-j@^O*XF!2>a_cP+m%UF4Trko{zkS0AO|DodH%8ppFW(lyDSk?QFE-BtzpAp zr{*RVWF1jws6fMlC~)dk<=R<^K`QA3i|KTBiksXrhNsDE_{+!QfKivRqI!SD-6i#N@#yX%VW{HcP1Kn+w`X=CsGBF6A``ya=scQLG8z$vV7jfzP_N2A+Iyr+DTE^g* z;?jZYr?l@E4=PhFS64x$e<=98L-53$NUhdP{YQk7MgPDJDSGz|89{OqWD<)tpMGt#w2>!EFaB$b~M z_Bx|byN+qH&_f!+mX({XpMhX5pi<8S*(Sc2A%VHtQZ&Jlk(<#cY;vL@ZQRu6Om1_1 zRRh*TMN{)$+uew!Gl!G9RFCHJbOH)v+#|AiCCQ{%K-ps=Duju>3C9u$RmjlG0Ny?_ zwi=<5kA`}+uVdJZ#8qQN2V4U}x$0pU zrhJuxd05d_^nZ6eVA6^XU>NbQJ5ybck3X-XmpJnHUDjYl0(p*KNZN!(vvqd9t%P^* ztxuuxO^>slz=51cn|E-%njc=)hT6mt%mMIKGX(XGpx%@ke8Ta8em(%>ZotbEJChm* z7ROJm)i;E3hMM*&Oi!TK0g;pQKOYUDC$LZH`;o~ijH^k%YlCdxluS_Y?o>v@`+k0B z0z$}J;Wk^HQMH5KQ|fh93&Bj18izLP6?vt>-w?Ov|FXiPzPS|dk0vEE%z zfwkR)ogkgjdiCv6*=h*+JOFg&>TmuyZIZrz_pv#PuwwVE7-eE`8M)3XWnO-q-{Roe zy6G-@41^?mQDAP}Z5=EY8N^`^L+rZdty)=zU<%sV9vU&;`ZeP(9)Kkm5G)^s0_0Ak zx<3q&h8rsn1rd%mS7UyWjB$p4L1xhosNzeKh4M6$%!FBuVYm3>@mftINr^rj*$)5o z6Hr)sog=X-IaU^%u1H+lqU`dFLe#5Eqbq{za<>`~LGN8rtx!y(+MjL;74WqQLnVLa zA+{L`y2Gv$pt0eY0#9DBkB1WMH8~e6+@+_rMUfA>-pp`{4}=MJ#iL8_GHS6tZcHm( z)xAZ+B-x>B7HveQi*I^n+V-FQ;i;6KIMkSQxE+Jk4K}L*rce#fafp%ya{M=IC+*|R zSpFI%DTDF$?>WKwvpp2w=L>da?cy|IAf1t0BhG8=60H1q2BtV&*t3h$JA~!_0^EoZRw|O4{(5BP2gv( z{~LreFab`8JacnF*1RN?XibkN=4d;=ibmXOryB&cLtNboU1sNgrw}*w49uEof8R8E z$Tz=v;B3VG_N3#$snyLlXza)|Us~islt-|0q&~8HExZ%o((6On_mdgLt2F!7&&;Kg7#yhpH6=K00xZd7AdFmlZsoYl(m?w+QRK1@B)R}bFcBjD)^z!Dmp!;M< z{0iuh2q$$z92TUQxuJSTda071h+)+k=e(le`7!etts^1C=NM|1R@S0;fTQL8h{q$F z#kP|PdjQPnBRJtRKGBB!n!;;=f`KimE9My?H6ehJ`Z=*@i}7G`{Du))isfk3KPM`d zBmw(k-RqGXXtmw$s9Wz!tb9u#LF=dk21f%+Q=s+$XjSkl&K9w+tn5$g?vk*dfg_Wi zDoi}aTq|-qmQDq1u(KaP=<1WBYOCLMHYjyJ)XhV(Sy18tRB6f$tNE@z?-((du9kO{ z@;)ndfP?|B1bc(51QBA}w7c}y7iK(XP>#6-3nOeJLwPm!iXhy_R64$fNw$~m$*+TI zdq?h*giir}UQpuRoI-)+4=<2fqOkO-2A)Dr=fRcP1R0(sLz+^$tYwf^61sBd=|OqR zJ;-k1K%71}v>#B*qzXX5F$7CycAZNY*q_MtnF+K6d6z1t(juXzIG}o>mm)=$oi5`> z8Lkm0R^@$a9W!v6wXS>cPNZyV?u~ZR#z@{I}6*6 zn_Za8e$r4fpbRPRRhJALen>!)t#P`KlNuJBt~2)T`Z0=m@*95w6(M%+MWFS{pRrCbcHf`64|`m)_X=%|vD^RyiS3TSDK%&X}s5H2T=s75@_}E1_WN;Z^B@MZ`!6v+U`F@)v_PA3Xqt;zH5nLvf#5Lm?(*=^XY8JKo1YVzfZm4LLxzpI&uW@nRt|J+wcJo~; z%;uf<;=dr(Ea08Dj}9D1ccZIYLq~9zKQWjnSko*?uKHqTe_O(yE8rSf{7|iThc_Zao2CK}vS8r0(>`jyqz}K3U?96YThLUXFx+Bus zbDt&JLH&6Pn=(o(BxyG35C!6@fvL#EBEt~iNQgx9a9rGhC3Cbb+b7NNeS6xTJa!;eB zpwAB`r$ z6;|_L`u;K*MU-%u4j?CMs?$=>11Bx#CRbyCif-)g444vvJ5%5u_(Kw~X^TPhZKO;} z**urM-p^R^EFybpy&IG*<5X`I!F0nKm_<-ExoeUW_CwZ1-E(S|B((-=;yq?ATV|QT zvT+D7mxrDFB}OIreK*uzixnS>U9b~HKcJg;@>&uBqZ}fp~_{S28qp1KsKnXmts9U_LK$r|KJh@hQs7 z68ibWv)@xwtmztNA)nW#8$)$JCvh>dLKl1&mSq-aVos!eD?L^>SC0)TG8|xj(B-GB z74LbaaA(8zXUfBpV13ImOqc2cdp)3k=P?Y7u!LZAK_V(CkCtIBfa0!`yCWMq3TWT( zxN7fqTdnsuWIc_@N)`V|2gu9y;+o*;2+zogkmUEOu}MDh(K3Ats!Tr3T$O0|IW^8I zgj&d_tvFHeoPN(-Zs@Em<6hjseLHwZn<>HMQ!B$7fSfh6XY-lb^dUTX4Nfkx+LwdZ-QNe3vqva$EA7-S;llwu^n_;%# z;?C}J)`Ej3`1O#vYAB1=y^MmzyFm2je_q({rOGWD=Tc zXQDe4{DSDq5H-$alLXkiLU6%`J15vTrZG+>{+zJXph{4QZaSVzb`Ukwq4@EtL)IMv z6e!a}GrLr};mbn4mokNE-#|^tqoC+aw{h5p;Fv@iXOuyJ`p*$W_<|Mfu+&PQY~y!D z(-yDH8UXo{rJfd5qQOf@msEk?DR>EiquKj7w24d*#Y>S~1}Su=%l(T=O-&+f0Wt9T zfqr&qNA9dDL_TgXgKw>as*U)56HlyiKYku%*gD)2vd9%z!xDB2Y((4X#$kyygB80~ z(aEUYrA+O4adX+;pG>|syCz8Cz^~(zCf01Nzrv=t)I1fGcdU0h&Ib@lxGIiM2pH#2; zx0`f8D%Ii2+IIVDAkIMHe@abF%1o(s<6tdSr^F|RJtoL9haRurZW`%J{mOMF*-H&F zF>~U+s;+QopH_DJ)_a3jOOoZf!EeP!k0;CQOuv(Gfsf|hR9zwP0R zL!mTssR5?SE*vFQ;AskPX82v(^h>uZihl{FCv(4r#3e-;;yejcN^?UYyXz!O5E$ZB zbBSRddwE;|P(1iZNP_JR)<37)6Jld?a>_Uz7}WxI=6p?HV24#|o&wc`gWHvau^f9| zP42ATEJ-O8srmFSF}!{9s~;J`YMmC z4%{95+j@pPXe%d+u!`vBAH{`8Ml!uw-EAP)$3Uj-f2H#!T$^5TVLnGfhO9BBy2hhT z_PE=LmMg(*gD!Q=_eXQw)mF2Ufrdu!gs8Z>3;%8EHzC@-F*PT_m;+ zR_+{bA}N8Ih67Nyv0rnkGa759w~7I6b%qFyz4_kJ|MMCL_>{S9LorF7e^WL$b7gRw z;;k2VG60*b(6N)F#o*u8UQXs-b2mDO^~3h7Xo?bj=Z#3NY**?d_2n+Ka7#M^^GCW`WYaCVwdiFn)>keE7(%BxqAtu0&n^&`vqBx!l z8VUC5rWt)!QQw&Ka^<+PF*^>b4zT(DgxW`UV9# zDVNEL(ui8qCdw7<#(^zjxxdq94E%s#yYD4a0+q&g0US-uy)|&NlAlQRdL-ZwzCRxy zLQV?R3;@#81u)!7Zmo5u7wzgq!W=C!UTkD!l2uN>Pp#}8!4~L#Lw0k(1;<%k((Pif zxqVOEaRHyC+jSSDDuWV)aP3uoemUtT?rQ@jk7G}0ug%lv&m$&XFJRcYz?}xDox@$C z@!^F7iypS<9{(mL9RGlZTOFi6dRv99w;sdALML42x#l;I$K*+F!fuL2`nI*?vzj;X zPAa)^mC4l^Cu66w60wp{QU|kU7EjZxGi=+rOhQcme+WLQF%p(r;dTTSuFsSsLrrV#widMA~T_YrC}cX=U-5S0FH zRS(|mSS(Ne0wOV&OU!9Qx4VXGfL+K(_;MCJ%ySNH4*2Od(YeQlx0Qo^W0KGRNN@Rv z>nm$syxO2s&G@^7m(`WeW*jZ=iXV^+Q6mV$Jpj=m+f~rKKLIPZ3Ew}rm1ug(O9e8y zb8i7Hm`p+20VS3qy#JVBZBZ`Z9zcsN7!`1Jc|$bI0eQoe+tJC9q^|Kl&v6kaR!u|x zvun}!OD3I=+uVekCVf;ZP5Hs5uxc6TX#yP8OPkUDABe_?Y?qxW(}c8aohO{~mc)x0 zlq5$b$evNkhP@*!h8j6{;Vl74udhEwx7VGAE%kr)-#ht0V?1W!m#V4OWjj1aW~E;h ztV*@8H|I4a8#?Xff`}AU%NDo zR%?U=gFb+5b@lQFbSXmOOcDN&mgMGO6kvn{9{_zoEMIr}3VV9~>`gG`@qK#zqP9Sq zp!~f+DC_{EFCcHMXjjmAT|_DUJAOT)c|4-zlK5m29zT!^q2e)ki5NUY$L>M6f9RIo zjWZt8P<9cmqouU*(wn^ET1G^CL`W1wN#sRGYpt+5nSiJ+5g0VU{l~DNV^L+^28;gF^I9SN%Fp?%m}!`ee%SERr% z0R}`VS{Du^;jkA=WF7*T5RxBDH1--u3_FM?68FzoL&)my5CuaKtzX>0$=*GI_&xZK z+g)LOpO^~<5t;&p^N;ao3@68xmO?tt{Q% z49gq{dGi4Lv~IZ^mPhn3-^lInEg)a(KzHO-b5UTQkcA)7ARpm&c{pH4W*?HxdDKrRY8_aSm>~bm<_#gb_YGRSf zRAiWD(40t&blzqD6+p-(-KNg{SZbt9jssY<_G5|L4K+2zZf<4G-2EMVIPRs;82rl_ zzn}9MV{b}=8X(7sy2ksPtLT)3%9T=alGf7p2gz}7DVFOS_|W%I{KAx~oWdwM2F}R< zCWS?gnbam&dFt8@KL*SUC;yMJcAcnRRrV3zIf@s=?a?cZnwEuRk0DnMyHgZY*iQoL zw324Y5q0NjRH8w)+EZZtg(xOo^2konwNIj4M8fnt_>f%u?D!A`ka0k1W{d&kYp(&3 z(gXKHK}xb+K-k=XhEz+o)O-ul$n=ErZR{F&#hDCpJ(pfJIb_$0$G;tX&@)|>CKgCh zW_w#$QeHpTSNGeU z7$B;7|6{|N)pU{CAeFlTn|{~2IQqv*MH)%6Krh(6P5uwCa z7iG+T7%Wz1Yy+VCBningL=584^7a!Fwvo2td^N?X04E4?`l;2bbEW7@slz`hIN07*(D*}-``1~t=%{-bK#PmTH_ zS+sgq1OIJGtFK4V3T)4nC3=~gi#E+9mRjZuWO{a&cfhBGGJ++0z_P*sIllG2#q0oC zV98Pv6^TUdb*9SSBfF5uW_z!9{&lDYvLi&tQ1rht2i}UTPi4-`t`xx}i!TAwFDeX8 z1|yQfIjCxa3Rb7WJ;O>yoe1``!Ds#h@5k~0j8;@Ig#+krgoiW z!onFshS^aYb-HJQ-y$mWj)rCP4HhNEZXe*R(>a5LIht}t4}@KA7prUfcZf+VVsND@w)Qe9*gD&PJBwXggQI1^;{`3$_ zYjrBMAXpD!>hCPFKqBYVO#`Bl85Xf1VKFGI%hX2BcjgL<5Q+Qi5ooN~22A`;MP=== zO`~678KzG7!9#laac6hisy;O*mi4CrJL=9k4NO`>SFn=Oo*Mc@qs`1Cj_ZGny(T1d z1lty%A0sWC4__l1;WjzwTWetvM>`<{{xWeERb|n5(9R_`$CDQ- zBPJ5K(iDd1_5k3|3IPKW?AAyebui*Es|(DXY`bid+yz~^JXBy#vHb9UQZi&~mLBD%|gr61U3n)-H+*lU=P*4=1AkONTw z84s^cxBOmaO1Z~?GnEiVXayy-FXVR*!V*lr^ct>EMcc$&dm#=}To(|%+C5{^`qttiXyP|DqvV zuFJM$C1ppTAYh8#gt=!iQ*~Qw;jo)gr*ftWP}Xe1Jc`U)Uu_YwFX22+*CL?Ox^2y}7nuazRLvvMxk1h=X;+*I z5}8ESSXJjyuC@u$m*7y z1y}=pCtCjc3dI}8b5@eJln{StWIlh-xcADOVYt(5L#C6qxEfj~OA1KexWoT;#eS>% zS_e0Kr9#9^*Vc!iH`96E7%U!JEM?#*4sM-KyTC3E1Nw?B1%K35Q82D(=6yil$MVVP zpK&vh^fp8Vsn0|09?$rI&$j;)HGN7`{vD+$nfSri)M35w<(nqwd!xlmSP#F;)s(v# zY_z|U;??K$8W(|uVE&lY^+5Dlr~(oxxjhjky)v=(?cR$b_y;1Pkw7ThC*A_dE$-k2Z;sA3q4 zyEfiBB5?P!QnR)n zi@35Aeg-80C+WVb z+~m-@st3KTn`KrD6EfAS>!FhtmM?r)AaV4q03k0BBS8qB<&?BF1F+5smtw6c{vk+W z>sq9~Mj=WLstHPoV!+yM^1yBfACRmCo=dBS>vqYlL$pmMaHLSq*Mg_bPN0tWKr2NB zi^p`%!)A18m~jrWo?xIHyS19WT#icK(^L}*egV{NA=J%qmmQ9H5U4eXs zB6S>^8zwTs143rTgJs5Yqk{r(>~D?A!|?Z87ifuE2M=6J-89q`E!2o{eHVlAaZ*SG zurMLfHThoven5f0q=hr4^^xe5<6~SS$N3Ogl!rl~9FIR4!a^F2JQ)|QxDU+9h!qxb z%XoyzW_rG2>o^_#=uPVxBfL~vgltZ1uyEjE?N!u4?gV^zRDn^fSKP@KP1iXgOxu!qlRLTtjVBodx_%)6;`!yH5cWKrzvPd*Z{(ZQ){5KZD-=uj%n#O)9xMU)gC75 z04;QtBn?LZnSk1%j2H#b)L}zKJ~Y901hwI2N-juVXOLIpFDDBc2HIs{t80ViLNSJ_ zJ@M@{Ur)!Y-e3;Z^!5;5=2s$ZYsACzK8RvRPRxgywM(52rfdX`_K1Tfd=R!cl^Wm} zR`ucfjy**2sN*+ybEiN`$`&PqvBB*H+s&5-IAf&F**#O!o^q)jx~cXw+^UuPO)hE0 z<(#Vl{?C`hG*O{+^-8JkO?6*ez86qUKv5qyGtJwk1hW;;!`&$G z=A2t(Vn#DlGiLZ~y4kqbwp{<&SP)tb@a<7J*!cqDInbh@w&72#6eb(4A=9tMQT600`5FFwrvLqn{7il(@0$n&I z4`5kc(L1!>UO)noYV65plc&QK8%5qGbI%3T{?|P{d3ri+bzODwS<}Tqx~A)`UPG&C zzJ^y0^8>>QF7ovQT?ZM0ky{r0-j?PoN|vzHB3I%*<|fJtUpCCacr_)LfGJEHBv3ed ztS73eCjAuF_7m-v7?!Yf9kiji9>@0o*|V)}|0}zLBgImCzvF49JzUB#l)?U`hH-qi zZ1LVcZiGz0Yl0?@-5urP7N=_Zc}A{8YVF@7aNL9cMw0hOZGQ3o2s}Dgq0xZdO;)w9 zwzb(71uNAgA`*6r655trxivnWae%~TsQa-1XH>Z{B7SEaF5D%Bm|N}A6S>1*rsP5c z8G|Rzi?$V*=fOEMJYksUKOnf4i6&TK#|VzoAd0a20GGC51MTaAU~D3GMu@%O9;dt0gyq6zxm% z_8#UvhxW5GXB2!b)%X7)>WpvBa{qz%Tp!Ajhcy1{mB*T8+PgacaMJYl6e0r*lNdHj zS>~aHJI6CTwE~LdRSgW}jO6SAPDL-k;J~0YkhXqreRxk{gPhos%A=uBE*B zw&%7=-G0vGT%_z`KIb{F)>k6qNnAt|0JXKBbI$0ccpwi9haHyQVE?bc=j=A(T>W9! z7MxS|sMAubIdVcGxrAx{y1-0tTC(S4LjERlQ9ehhHwWlydz1|}!)uvx0nb#B(uSCNscH_iRVNOZ^G{Ds*_etT4i;s+EU3(O4aI32J9_^7IUnAetBYG=?MlWV=O5oIE+9{ESd#_L78Q9T3J8kUxmpP!wo z-aD50XOpL=#DmS)qNsbuWXm>xnQI8V2T`Uv)>=>EAlU>e0UFQl%AB{b{hS^9Ab{O~ zOMzG9cPmk{-wiK%OJ~q_K4kh1uwY-@(!`hpdw;9E$I)bywfEe$Wz4uv7Ysy*n^Zz) z)|?pmKA<7bSee>qbvQEpTjy5vOaQ-|^F>`6d$$u#)w;iDjC2n~T`Bnu3N_5@?3z_5 zEoqRS)=QiGdZoYh(8PL=EKkuSg^RpJLQAByH{-KUrm?8I3zkEsNNP*=DP=;;O;y+H zf+c#&w~oknn(8>zDOss+bS)05?*hI#AT`paPuoP^!|SeSwTzNl)Q!3^#y+kSXw{)= zwNj0d6aXs)Tm}fNpAzz>mK6vUnj|@XK8T_h6v?ntr6}kER%=w>x(_v>{wR>G^lm5t zh7RuX#S59RC?!R!1?miy4i?aSnXU;fMV;#a2#*-AbDG|C8v543{MCA~O{h}3u`g!e z>GbK7t%o`HKm&Brcew`QHhC6|yw$n=W{q#%3MDmsoo8{%^c$VYp&ptTnX4*mc7|U) zl)PdQX=^8Qnn^4K+>~`aZRa7vE&guJgvi){q$umz*85U#Kv;y$wzNy|cv$dhZQa%u zb-Q+SL{9L|V@XOOs?)8fPj~axhaxWV2cUEn(=THBJS^ihW8vxZ7uxA3`HE+>%+@~n z?izanQWDGWc_~0+^-x+0G&;PlZoB9D3dFCe_MQ*P8OrB0PLsu3F?yRP3Bpt=aPK+zSUbbSYsBR6({ zZR8sIAYWxvRy_G46Um+Gd!%8e@5xqAb8_+BHF?S9UAS^4R?KXpR8Ui9OesMi9vECr z8(qRa6hitiU#`}$Bd*0MdHYJ%O zA+KPh*PNsG7thJ7T-F#LQgvSAr>R>e>q6JK&<+EbH4TTBFHF`pd4(pE zKvKyO`%ry;bViOo%%%D#ujcY-N-kGL#uHvCnIv~=r8E72mnNUPg2+J@h4=k8U%&kN zIf%)rK%i!J?^MDw09noKs$<$qF%hex6f5rFufQhp;a)|h>wuBprxlQmHEJI*%hdUz zC46|@FY4-9pqFzZFi81K$5W&4ic5Qwm!vJG4;ue>zXu{kcr((=%uSt`vV7 zSG%87lB1_bQ*v!2p)_%3AxJ=b}uQfjNd-yrUyoRUrUk%Z%|+89a&7Oo)}DE?>rv zlX(KsJ^7C{BbP4?gLeRyQ!ekk17&M0b3BPpE>`p>k&{c916$VEc6=ic=4_AXX;D^&m^l; zl&tEk;zgjYDJ*%lz#szOR)V7mpt=z9lA+r@eQq+YgDVi0YcH`k;AcoRcO~d{NMVtO zDW0G~GPjS>d&)C9;~B5kNAM9xL%|SUbsy`t2O7sHaolnL)V+ z5WYJClX-AZDB4ke37^G=<9cuS=H2B^XSL!a@n^NVyZ5YyAFf$>&l46<7_=wuaL4ub ziz)rJE+!?qCnc*!o#@RY<~+1$$M{nllgQg?Tm{;^M=mrk0@`YheZf|dECM*qxc-?Z zO!tr>!q>#Ju_Buk-oVWO58q8Q68t3wpr-IR@<@-l zsamQb#Q1WWJtyBtH0OgI%(yUxsQlVXNx?&4i? znIfpJ0mXeE1LWf@$}y*nPj=WG>*ksEXkWL!4{49EFe?EOb zuO%y|`07t(a6<-qIF9F)LlT`e&ZA1tq09OxZt1T6dI z$2JzX&=)b%Kf11^LyuxL+PWw1m2k`+&L2d1$MePHfmh3k_JN*5Y4`Kr##F{Pj*g2o z)$Z3?vU#0hz>S;|T5uh+cLvJt8L4^tmr$*aQ726hF>O3xmAcekjbHTYZYc8Y9#N%{ z^-7uwh({`6#{H+1vz%$V;`t}T4Z<>}5m~mDA9l%XjB_ zBOgKf`y}=LsB?3N;f#~K~YSX^V4<*$aSD4U9pOl4U$eG@?2LK z`MO~Fbw!gquUF(zKKRwk2Rk@?&9cD+Q=OUC1d8DX`XzVcFYLF);KDbwh_fV_yF0Rg z{Td;xFqpLv1pr$P4gscU`@*YY_O9mP}o}vV^a?H+l&pg+DxfIwk-6 z;>~Lae=I7AP@wo$bwi7=2e3(9|Ks}Ww+$#t`$^Cm?Qph2g#FF@bP3H8BZ$A5Jc7~0`LmNrtIi?S6U2& z4xf>X-!U?me=S8_G)Tl44(p|rkelJ>!xu#~*8Y{mpB7p1_epMFPjZYt(!WkqVC+oZfZt^xx{poEYRptXKP z7mM1gsFHE1Rwp2%ZQ0@`E?cd9iEF9OHz*?lliz@b{2Cfc4QQ|v_YG+a8r)0Y8#7~e zQd&WIl2x^yCJ4h+ZHAJvRS462Maw&-BWjL3h5Qm}amBE#!aaJIR&?#=Z3WwUJe?jR z3lSxe6)p9iSKsfmHpm-VNJQdSLM+s}5CHMeWQ|ZgQlCUk4&6e%HuqW{Vih=+dm47U z(x>v*Tb{~z^_m+{*2bjB<8jK(YOq{b`c;lrz4i&wk(u=|>&B4?c2L5mm>CI#X+EQU92p591(rf>u>YkM1T@d&aF`U5iM5X;xy+jsTlaQ%{^ zKNGFnroGfI37I@TU316YO&w7)ZPmosF*6UCqzCe_WZEz8NKWeX;Jx4l%>|;K5rPos z$tOa_So#i9ey#r`EZ`NEf^u0huZJ^xYW33yX{pfcQUShHJ%zG*i061x~uQLTk# zs@5B%V*)E&i@MZ4FH>Fw3IrzAP|9m_QAtX+P%232Ku{t?$j)K9+l59qE|=kGWN)8P z{82M{&r>7YfQ!2U5iF?G)fm{u?@?o& zi6&ya*>nd~B87`E@1(G^KhVNLZnfISvwL?Cnnr3zbv zZnP6cI^g6YHw2)Nx_M874B&5V$@B*BExSi&%dg(qvV9bFQ_q(=ASW&O0&oA#L1YfB z?ZffLJDsdq6au}wpoOQ+HTxU#@Gl2AP?Y|9)uQXut;j9 zh#@~#bJoNwgiCiuEZv9%325iGVr?oO@&{U`7gvM%V{MVFx!)B^&DTVIPiFwEj5!P* zhu5Ze?3eU{^%>YIp;+05C(FAedc9me^Bs|aQ)~;T8d}P{R3Dvck508;hg0nmooa49 z^cKG{fKz@e-B`cH%ZAgTc-u|;awPGPBqQ{uZD!Gt?G4DTzv2cNhY1ulC3Cw{8EkE0G3wY#U)X~T*VN~gq15v#Qdxkmv?mkG@!eiYgA=wy8)^dw}l78K?<}SM%suj8sJM+D;w8Ya2IWK>l!L zjHvD1xDU^)CARH8Y>5^zyD%(#bOnqUs!<|HTg-?c8{JYeS+cy!)(4ci?8t?Z*_*39DUG`f%%={7y#-r~3JW2QD-)h&`8~O(r409& zj2E*~e$SsqP^XKHhPTv3i=gH|`PENjmGWqwv|Ni9t6h8d!^e?ABn?HVkdU|s5{a0`P zdUN&a@BH-V@AGSILgAb4xL1qYX~W8+soUUW*35V*`<^2iiMb9Xy&p<=5-l(bxvRGY@u zBk(UA0F0eM|Db&-FL%jeT(Okbs}VpNaKAeu4**iD!Ea4W1>WX&;%fMEn%tq5WQTJK za+8y1dvv7z;44N$wm>vRLm-4mnHq0H2;0;$|Dp>uVy`yEr9?^IfS7Rb=_An~0Q@KR zTFE$GXH@8vj6WNrPsC2bpJ}FvRH+2$nHo-J4t2A#2)I1bmSSM>yfS;$I0DrYQg1}* z56q&O^c%csQDi&;=zGvqVVdo-xdBYy4^T(GmTHlUT;JoRs&|ZS;DbT!2XGW!n7*yQ zhh&0lJCcnl2r^2IdMRsAHgpeA{`FE6b8zolaZ_4E^o`A!^DKTh8USlM33aXm(LT!d z`6OkXEhU`r;btP<;wsVF+uO&RgADBi#SM06wkEVG^`yDKaTMQJV~sxcl69hyTX+;& zJT?PY3?za~Y;EpF4FwC;gzR^VM&ua^(gr4pm}E`Io09K?zFX2e5z$B>cz19M(xu{Mx`0uUb=(+p%K87K+Ld z3(R54A)mVj;8!Ei+sIS9ta^VQUJfP1VVT+jEKh|K*Ku)mDP3(|*Hy$fcsrkoeBt74 zH9J%t2RRh@yyt06Gdu{e@q;8^TwfwtzF1b12L_&9JkuLqV;af30jk5hOu|7^X*(ZU z0kGAKrKwYC223drIaX6ve_Vb0;*9Vqo0`A8Bn=sJ3{+x)G4?j)0~S1V z!Lc`cX{vEDbZ`GdbzD99=gEIK)eioAt4^F!-4IbCL_guib=xv+w^t&)E?6?v)p%}E zF(xdp$^gRFm4Kitp5j)4Hka%lCVD~_dDUDnp%u%60m~*EHWw9=#*(ryCMRGmG{spf zA%O1yFZ9y9$O-$vB_N^T*uvCM3^um{ih8A52?qGazfgr;l3XV+QViiqkMMp(uzoE8rmX3IV0dpc@;&L{vCLjt9X(i5p z!92ql0hUrVC2vKfrxgRTK~#o%YKT#0--gVRr6jFE8gH0s$R`+WnN`x=s1dzs691DX z$`;0=Pu2`OFBL(SSG;5f*8q!%!$8+LF6{P@hYiz;XD?VKo8U}5&lF?S(u=m-z+nf| z5Tr`Av7%owz)6%spG)fW4RwBuiw^6-y%A}gKd)1|t$DafzQE)j3Fa-lA~`KsnpJnA~yE!08&vCnE&uj4js zWKcwTDswTeSkQCwyOw~Xg<-4FV?xe}QSqmGLJW3P`}P=zHR-$%B(r`kRVNO`ITMZK zuO#sjOzI|OQZ49GWEm^5Ks##xam%2EVgeOc(Dij@%|BLE$$M+1Z(c?|qY}|1O~nlHDih$G>Azs(C#}%hv?ze-osx(-45+| zv&Gm;rrs1y5_&p?GJ!N->ma>q$;)(7(6U;ijpod@KANzbHW~TQ=$MdDDIxc2%{L64 z@tA2htBp;7*^BOHGs}nSWlWByxmvw+OK%v9%O~2)_?Qhq@8BMEWB0~j3ccId$l#9M z0;oxwSMDM=DOM}nfl#qMAgvPV!cBs+vM$&k)Z5MoAGXlC7bUsKCpuZ*I&ZYxZ6b)# zra<#J$xaJx&{~)L#nq)|(kHpdk?rBVae^CK(t4E}Bx9uylTRLd!aA@_Mm*wqs?GyA z7Qv7q>m*^3TS=P@-Ac&K5uk@>MXO}VB#|O&xf8;3SN4MhR08WOE^e2$)k#<~#4Qq4jF%iJ1I(WR+83yQKv z6Z}%dMKsQ6a`!;XR06;OtvFU}J430>6|nN-6&Y9B+okPfTpX&OEr~AsE)X$2;|PxUVmU&rvCMD?11Up(|f+i zF}t5jW17^-H798)iXv7}QfvLR+j|HDDeswtU?7w9<15eLUJI79l2+jO)Cit@PUuEc zTU%l4fpC`rI+@Lsj^xzNv}+1rPE;$vY~IM}0$`|9@{KM5Z_gK-mpxz~h!cX)hJf|~ z^xS9@!IYd_WW~~XbL7Edd6P|DZI6oE?kI{cN+IQ2Lw3jvV!9A(U@+Z$)Wjb(@xJao zYT|v@KZ7Rj@UX4BYNfNbanb4Y)*G0-6_pND0Lhw?aW+u0p`}saoey+9%~_@KsEkkG zoq6wHOBw(ncQ37$IUCP#1%sHiMXM`;M`{zV0nZ;x9U-88hm7MQ8bC%xppWo3$N1TOYhSfO_|LLt;HW6;TWva<9b%LMWfP8yHw8N^; z?Rq&U?^q)4S&0ka)rW%Sl8gLH%ClIHkXb5?QwU1cwc`^;dLb=C*VDby1E8_L6ipo% zT9oXbi#l!`bQ(ilp@bnTt*bBj!d7-Qp~?_?Q>O9SFoqP`DB_UPiabAJel z6O%DwkE`(za;>N1Gh@k?*|B813|aE2CTFy-JtU)&4c_w_GurXXnN|x;W)bz*!|z^V z4mP~HVou7u<_V*>L<@ECd_{A*U=vGr(j*6Yt1r9PNM{AHsrZlk`ljsNwrn^CtT1V9`nMQc5q2Q@Mu zow1DkShI3Xlw4@CLSkch5|K-u>f9CWt&7nF#)>K6rhD~4$;N+FMD$cWfO%bl$uW)k@@Y$qQ4y7xuYc4dh#z@iD8&?I9|=m-!ia zD=PKRE65-jxh~c9QZV^eRPg*qnI!D~4pXH4dmQ{~nB@gX8zbA%w8h3!+R5jVT;@cS zdUUtg0=kjdJmX$yyx}7$_Owq=xF}mQxVNTSA9x_uXY)cjU_XTj5}OtR#r$Z+77>9~ z?Q6x#1)#$vQRT0iYvdV)*Ou#gT=>1Y#+VoJ3Kzg?-)$C6yo5YfNgdyF5QJVsqSMFG zY`2)xP00-#j8eyn7K-cqUnN|?2Ks*^^jKcXDY+n$=ZlQFG>lo#d0jMF)gq-Q{aEvR znla30q`9s6CgvORXFRvSBCkp$u(^FlcWixh*5yml6pt?RqXz%8#e=L22Qdvt@LME& zG0~Y?T@iSkXnRP#Y1a@`ZKr*aQFNz_xAk`^^c2d98mDWAPU9@onX?;4=qQpq8ma+u zBQ6PR6hr8i6xm_$2T3^@yU__O5bm~QI$Nqk)JbHH+?IOIHj6O3^|zX2qbQo9)CtoY z;mJ_{tEtiwuE*D0O_)Z>WFas_w|5XmNH9w%X>G#xDpJ&CA*8PR)ZH_2oReEbW0M|= zS<7~VKnkf&TaKc5(Z*=g4kiGe(!OQUKM+yWO(-d3#M%&b5SE^E_ zDmMnK677yaI()dYwYen2BHT{BbbFSJ1Cwztz~h>UI@jyopnUBQ@}P)?}jEZ zFs_kvHBK5az-R`J|8maC66bEV?&n-luSo^NnT9u=;oSAdaE{52(-FFqTru3XvZ56` z!+MDG%#=-uK7>{x<#QcDciiy_@g8fhLiGN8$Kv7tq=48D)?LSltovq>Dz!l>RyO5QO__5^0f zb3??tg15{EK`hz@S(CQ;@k>4{8F{g!dCoEih#6gDgKef&MUy3-1(Ybhm9gqrlBJLp zSHSKRhZ$=Wv|m=`Fg)_uOIx~JIy z3=OKYa5%xe{zW@nESFratk06?&biF=175nkCEZMQvCO4-AgLe^3<8mqoC=u*GNxjV z(|btDOT?-u#fqCKd9x;V2H?$VZ%L%~3}c#ejHwk%XC_)Aj$dGeUCMLwwlnTYdacbtj)n{X!SJHcE#})O&;s5Tt zxADc+j+v6+C77f9_ZW9XNxLeMY(eGt`>NWQAG zisnqzGFy9>)5n~I@Zp-V54aKH4q1D9`>=(yEuNSO3#tt!K&8pJP0#JgYeDna2bO5C zl_IMbd=MJzqt?T3N_*E5K)(G`=~Tks^@!N6ME$Q*47lknPft=-usne0C35m*vrch9 zb{>0O$Lp6b)qHF!inUG*N5fr;ER}?Ps7mU6c_LOTCCFNOb3$`J>xxnx_9YCYzhYqk zOWX%Clsg^<^GJIXT3^R$m?ohe%j~!7^25$#x#UngEVis_C7_bGh2jKetap}d16`nj zlFv!*4_s`wu9eMe>~p;x@NfMUVz|>B((#eN86+a(XW3Jg9DCz$tnR6|OhDwGkIAWP zuZ*|O;bZctMqAm$JuB}ydpOq_Uy}!3Ehjoz`5Xku^WVZhL2CmlRTpWhb2bq#RiiUZ zof5BSklJ%bYM%ZjWEC4F@&t_++qer4UH*LzfW2{LgY>-}i-!JE3t5h(V%VB7V8PIP zR-dK@%Fet%8`nMDt-R+Zk&~2lo)1r2lxRjPDp?gW$F=3N<{8C3+&!Xy0{;_B2@S9e zjXg^yf7W7N-l8%Wr5zYkZphDnO?WN5Ta9tak^uT~)autQ`sSO_(+ct!NLk(x`toiJo5 z940Fo#P}U++AI>*xKdIs?{+EewY}-Ft1pX9I38(>FQGj#DMwj31Q6ySJc)r6|dg0VldpWBT3=S9s4h@)E7c>C}m13LXnvy zbw+)DBeecb$;($)?_RyQxOw&RIr*!819yezDY+4ix!9xd*pxqAP{g+F44E*Yc~eIb zDi6?{T;{y8Rj&QAUy_MjhmR<#+FHHf5bncr{Sx(a>n$Flj9g(+SKIVKLqn7{WlEsN zD+#s+tpO%N=Tq}N9!Wt%aW1T#hKLf2<>^xM)9LXPax#vm$3_s#Ot7Svs6LyT#<=yH z+6!+=$k!&h)i1H}jd)EPX4#am?0G|ow}9QMp1FX zzt$<~{?G+bYk*&ksvNCLY)OL$rH{08+o{g8G2f(y78#uyVyQ_opQ-`gNp1N%C+5ysHptx7e_lIb4N5 z$OaWhy7DaU_S~Ag+f1-1*~C_#ne&VRdbc@ihkdtsW9l)MIb+Aer?Yh@l+TgH2FZd6 zolA{bn`P^!5TLe>&;?e8DYd7hDEU3l*urNBT=IP3plG?+TpF*gZODJ4{%UY81_{Id zc>DDY6z&Co@y?2lYKUh^5Hguez@&8Y4<$3F*(v#s>Gwl=LsjymA`eSp%cMfz!ZQk6 z73;m}r_U7yC4b`a*rLPL`GT4_BE#I)dH>B=%pp=Mk#? z4m_93MDjR>3%9noZB8z)UtC|d{UTe&Pe%k$dllp%zsS&G3ccBntePxoaZy*xms|n} zW4wT}@uZW3ft7-_H-d0A(B6gG(0K8;iz`BFHF}jU)3~E#(Dd;dbDT%_P(Qar{|tpg z^lrc5^DRGC!17M3E&N!1n!079{aAkg)kg@$-|?h@9sS?Mcg1sFaaxITl;5EBlr^E$ zvJxxYVPMAOLEMVBV9dLmZxU?sLO})+dZ4op_(uDF8jyus;sZF69J zP1>^)5LR`(MN8jr=f`dT75qkCa|xk2p?dUiGncw7EJ$uwO6laq4>V27=XeIswIo#r z6UI#`2AW$h_DQuabgiSC7gs<TV5B%)k zTBG=z!^6B(_ru@zqUiG>OMdHKJj`r5uxi zd5rfPnMLg%WIm;0|BCT5DPyqLx~uLNU0FLBd?&ge+p_th4~i&`!}UI@A%qNyQp5i| z+e&A!89<{xY$Ov$Lsw2Ut>N{(KfIwX%oM<{?8H2Q!no*A`$#wWcwo3f3d!bB5-QE9 ztHamaEagLR=AFly9E@*Uw*6f&OVm3ky?Wn%biOdpuxn;(&af1f$hXL>>3gv4w7-Ho zUW5VLniYZZ%hr_yb%ynsD01UeYP*zn*rWeUY4!Uw>F>H#i*!UHo4VPI`Q=M3VVX=d zsTUcESY>2$r5*f4PcL6?y)i{uJ&1A)NJsmTP6Unqt(!zTtc`qHd+!?dUZ>vP&>r+l z#8zn?xdr`x5bJmAY{5@!>urBZ22dI7vFYv2R;Ua+>mRt}NAMi;fHq{PiEB6x1*4U- zmk!NEOl!90B&GNVK!wIJ5py>h@3BZ#mZ@10xmyNH11k)N+hd8Q6)ms~JW0hv{y@vr zQ!F|v%>fr;2o|%Q=+Ha^!FAx%Re#KQrLmtiLbRi6`&}h@wA^}v{4ntbx)3ECc?{OM1BB2XYKWeHz(i4eT&4^N*?$^X81;|u90x?HYA#42nN zOo&TZo{|4?{q@@hCsqIIC9IL?)f%zGIVCTh@H^MmN2_J9!V()fFuv*6toK)%o_}|g!kB!1iElWGJeO%Ttew_e=do^P%fENBELqj#uqiK zOENDA(SDtg5jAbw9wNP&O)(=~UCRWLBQFSSh6pEL>z!AYDo%q2kZBkX%|yO{Squ0_ zgd1$cQk%uQE&?8#_?l4(MrYrYwXD9V^E8V9j&1e!=*0#FjP2E_8K#_d6#lT9quVJU z<5ms5+oq)L=F1|eYSX#ePc$dAuEYdj=uNQol)TRg;3}U1?r{u|xt5?d+vCi>i=`Zc zb^AW=p=VpKwx4zs%}iCT*rw)m#m*GBH2}wLv_Q!hIV;f~6IV2fc51f45PpR+v^rzd zu9t)^7EoQ}K4zA3#V9iVQ-fi+WaR4A8;Ai1kywdh70WTXF%A=4Ex<+$o)Yp(iV6_@ z6lFa@(gK4+Fv(<=iNp^lfL9_`x~52;5_pI#8BI;~X(bjF1xu%dyil8Q&J$Wi6*E*% zr!1|NT&;6NmyC!qWeATyCy7H)pFxq^3r+&AS~%lXNy{|~FJ|r(v%>lWks}PP!fK6> zedQPT$>@T1u+FYW)R*k=Cwvq{yv@xe#}u`Lf;o6#cNafF{|J?Y!I>LLk(;e&->ix|}-{$J?$p=fSFT9;iO&Au4q4QloQ+3EQSl=iw@K z9;#O7A*yvAuwLguDs~>EX6Fu7JGZRcxx-rDu6E}()jP-5@7$__=TUi#2 z$#ctDK3Fx+gVpmqa7E8U)bxz6>N$p_uK&4*x}IC?i_s74R^2l$ZXB;(o!2bFt*mAQ zO-CA~T5FX9s;Q&KMp9O*qNUo$P20%rE)3lG_%T$cc3HDCl8XGe(&n8)qzL|07x5~- z6T+_!)6K~^k*8|;=Et$znch>L(OG8ndU3Oc5ffSv8GwA6-_Ea-jpNcikF<(#jV?4XjKq-iFGS@)+x*%mw!y~Dibk90( zij=nOy!)T8c%jrApxb?W)ES5l@;9^?=~A-Kxpf=}or$_K*|GS0#Bc!udL@kaZO_K( z5voZ$l@&q{+OaQZgwbTFM6d}6Igx8h37kf>e-fQM-=zpvnw^^Y&<5D&Ej;-ZpFJjLV)7K_-Tx|~>>QU#;JI0JB z)YXC%e#tbrGBtKRCrN$DlUR<$u`sWuXjX!q?6X)0p*OMG0L{k+QM&WA<-hDWFfG` z3IK<#mhlcbyl6xnE%(~S{yft5JY;v){nWc@j3s9)MST@n8@RXG#}K2^DmrANRz?|w z2I8kM0Zvy07-!V-2T<7{sZlhe0;)z)w82+_0IMx5$pgVyPQcYDqE2wuYxVd6Oz&5%S6%y9H;`K1R z+R>p7?r7Q4%tm>Hohborv~ZZ`l)TXg-pIv6l?*IJR#z4Rb?cK8FX)&tAR>~rA=#X! zqr0t-#P~y4KRW(^7;AAKV{MOAhw$3ck{D}U-_i^i1d)jB-A9I+z-uhvL?qc5SPHy8 zo~hA1YRaBin7NiDAK5`s62+rI`7z=)!q>$e{^QJEqZa-y)^HPCe{pp=(wg8rIbct4 z?)cSO6fV)C-~=#{mrb|69V7lkYxLVIPa949QXQ2|97puA;ZA5Jbjm_=_%_8sO zdD*3teKtyV+&0##IUjZkTG2GjD9L2WR&=zSrLHg4%v@>lvWa9S6)RVgi23;74>x1& z=>BPBv}JQ`9Uw6`UD;`9uN(krv{{0RosCs~h=T@TEoh3_-{ahUq+!`E`oj+8XhC^o zILGe3XuNi_$n;oZjCEzyEcq-kXGZK9t6cdkF~+(wj^-Xx+cs!MrMitG8M4H{T#cd_ zw#2yBl~JTUmYC>#xlNi;`E#Q!QhrMeNuu1!&GpY^iE+dmb85D%GUE*$F2b8J%{17% z5x5rJOFMI{QAU)l0$az&)9LY;*|D_?wAxlz^U1#vmM0=LIY4t>j%=h7(iC2@cMSG#Ob53*T1`gwmpcp0jhJjv|07Vgx|c0UqUGf#V?5(4uCu`uJ7sdDpWy@v zOp~HM)5NALwi4y~%o0>Tjy%|!)@W)FZA6l~ELmP<>yIp%+wqGJA!)Amgxbo6Xd$yQ z*l782ZY$Bsv&I~wzGeA6xu>NUzp3xqTDVU6J(uG%BWS+FHnob!0;ZW&guGGO`zI;r#LmK7FfbAl0?*bbtCRrL;y<1mastq zN=$Wa3o*6?us&>(J8TFWty8Yl_$}6$dDJcLn5QhYKSVAu%}pxR+bOvqB};h0HB17< z?{U>>;Nw_+%PYyUIZ8z|Z(x&4M=93I;SNnkaR@ld1DA}*rKq!%l5ShKX%)*q-oD*6UrJ%`@NvGv!BW7De*jsShtX}XOl>(rwq{M7ZN>R}& zdIfEE(Op&MmFgvt6j5ZW$~BZ{5xP0fIB;hp#G95xmYUL@|Mo=bzlq-M>WqP6W0fFT zeB6LKo#t+4DH;8`A{nEyB2S-?70>G`7NWpWS^oI> zh;10JtZWHt;3_Ea*=i}#$gKnvX0_f5!Eq{GZABkl4?Nwbf9MASb>Ya+7xC+Vv7EYEh_kVDekFkaLE>2R^^xx!3WxlQ$r<*lC0{i z;zedh$J>c_hL}DjUGE5WOz#vc%V0^@f^t9eJ+Wf+lw? zPsumZ3{JYv=}H&AElP3E)#7Anhjt)R7ERNL%WR<2QLNf^q-r^$V5GV^#D653{!4zJ zuoziX(cHBY_ukS?8{II|>Ff#(;H&+%4xt#;_Bd$S9);$66HszhPP{J*>dLBv(bdQ6b~d7f1A;JrB=H z_uT)8kdsISImU@p2;p@}bV@c0k+ruK ze1p5nSvch^fT;J0ETcV_WbYOn^fUXszre3kfBmBkQ@GB0QnD(L0@5wDt{(yY+;KG0 zx>~Zl;t5)IWjNr_WnJWQj#@>^QmB1X@{~zp`BSq9y(DvnKb>GA|aL2iBKekhL ztUET!(K?FH1r2fAITuIii^rfawiqOQv%w%~y5fQS=ud)0hE+0C|Tqs3Ai&!oRYMhvXt1b`(6;2XAXc&Z@_<#k(Hb3tF)U8h>W% z`_1q{?`WEqOiIH+^(ygp@#5-|{1r*CBa- zLo}heC>@C3TIsD?pZo<&LvM&gzn;?2<3*-MVsh+qCdB|Fchn@sOfMWdHj;rzmP-y! zKqr<(m+;}T$qzuW%5Rx{Ef(Aq0#nNe2bOX#9_^zX^sASHq|9FBbe8!&t?dPEA0c06 zbU_&Wah`Ii{w3Gfufda2=UNx=jso}=H2<|3rN|U^->b;zMiu^co*_I>O+ioxKTh8f z3|6N${RV@LW{lh{d2)Bhwrx8*cK)1m@3{Zl?Xgz(s#QbZ>8n$40o5#iKUN%xuoU3(BOcodZgb z3*y~$nKfgMVOb-GsRtwMdPi}2=!ONH7~;^xaVr$DEp9-oICM;4&jzUH3tX-%VbzBl zvcdZ#HO>f!PwAFquO^iIVmdA46y|aEC;ZEo_NJUnjC3d5HR!?S?co?6@3)tP&K(jIjJ+m*hrRl2 zfIBsENf0nX{0w@eLKar_idk*0!@#%OfPQgl)_t-?r1QwJzLYH=JFuVav(B zsI_VO1FZfCs}nX+xj&9*BPqEX_u)UyY}6+Y98yP!fd@%|giOg|ffpW*iEWQDMddEX zgy2ljMdO-7{N&k(=Avk5TJf`(|HQl3wHgHgq!!{|i=~gtblWj?zx1N$y^{zI0PZC# zT!waq0sFx*BUboJRQYbZ{>_F+tug~Fzq~3jP?+B?opg0wv&x<3#qh7zf*e#DMIkAV#`(uR~qfvzo|cZQZo#3jJf2bdDoP}&1c3qv;~ zl_euZN(z;SRD|>3t_zWGRBr}1A_eu-Yt4A%IWacfrpwQ9{I3ZZNIYuy=b%z}#mp-m zzYM1g_~y+VDR1UfK^|H#@RHy-I(QI;7f(+El{XMC)zV?F!|-bC$p+cwX1fC}r$Pj< ztD#Vt$@gdir<~v&m~Se4IkLTmMuNb$j>h=3 z(HrG|mZo4=-|!xcz1R~*^wV6-X%9&T>B;Hp*tNHfbXY}d|r zY;hpLL78cE;RHT*>4P}V-3O)~*GZrHPqFAO&_FTTv;v&JE!Av)G0w$1V z$2vqYY#&uquhye}JdA&JzYS2P6^dD+O#tR@IIw^1Reg;3Pfk~YMC9{vDZ08me>;Va z0+a30NMLn#l>*uM z-@@1IZ?0G+ge6#U=N}O{O2qyqr^sm?(Bg0sW6$aiYy~q`()tZ;hh%z6O!Ya=pB!cy zAn{XMGQ@^EXhW`m)z8e%+yn$iQV$9WAv<}Wzn=kx7@3JgwOFm*V$8zaTe$3aK|oxZ z=?_kT`#X6-zkU^*2@Kz_god0wT_h$%jFO8u-TqZJ1(z9y=HpVTf5Y8Sx1Ss zHFNv_a5A&0B2IhS9Uhgi&?r<>7m*^1d?m^t@w~<6E zx(&>n8ZgcBctRUkd->O2mVru*K6w+vWu>$$jSLsoy!M0zivhv=ZQk(#2SSbqLCR}T zbijY;S;5spm>&6;6KjK`juSX6tKhNG!Lh=f8?Z>OrwF#Ze zx=1!3r4R__u9O66#|YS|<`m>Ym5mRFH@Np651NpF}8_HJf!w$qtATlGf7n&>rSehS7A zTk&sXwY$c;=;1hivV4TcZH3^lXk{}Y!t{M{`%H>m1}$0^iqYKEXaC9zGNkLoGJqRf zZIH|#nDeC*!HL-T6GrvMFRMdLME5T{CceB6JP;pYND14G;9K1F{_X9|l)$vXHNDD> zrva2}#}OjuY7v+#L(apEew*+`;82;O0WOMQoNyG_nS_dMJ=S|m{kQtraT3nJKBl?- zi{V<^nkkjTD87%mdL!xF7?M@YDWgDsh3Rkq{=GyD8K)r{W;)=XmN3zkrWQ7zArrSO zPqur%)sDb@8bYmaZV>(Vxq0C}@ToMGvYQupG`GvbsW4SOvifdI-{>6=0sjz)$1K#X zanEn0*gLiz&HE~ixid)(j{eA>V6iu8~Mm|7jnN>QR{?7n8+}7 zc<1~mYNeq@gTzrwwePoKWjE6ld9E{FnB~!x3u%eKx|44YqFWYd^g}2G!P-lV91T?H zbwz@#Y?**U46V@2^bZ5VP16VtI_r$WFewBP3R$4b!jX6`m|BThB{QI3Yo^uADW2;C zc5wC(CE!Zl>j_r5hoJlp7sT&caLa&2D=& z=qb3Jvo*56G&wKScrEf=``jSft6kBy^ABPRM zl(~Kgxg|{n3&CBcZXD9aOdG;?e9o}K6^mQ#2<#T$fBtBmdmTDDY&@srC!za|G_J86 zo;5NuRQ_vjRn+2N2od!9em*;1brb)7S1*OuLXrc%)ULB+*M=}i_ zbLR@H$uJH)wGu?C3_36(mR(vknb*sOo9Z1})q>?04`bS@SG?}@Hya5SBe6b0eILzW zVVi|6?UT^8E!!?7(o3P6l~ ztx_kIla7dDxZp3FRt|Qy0naOxQTnqKob>~xe#DNVXK)Px$`%7>UuW{G+Ko>w%Z73n zC!Z16t=ERjo>tRlryDbPZ#DZ(tEXS!$E<;}`Chkwn%qA?8aK9V&(o9lVD-Jr;=kX& zeeP*vSA5^8)aM%#PQ=suze$6|tf!`#wT<81MiR+nb3V@CeS3Ys!@hakpV+_NAI|V{ zKhNO5?;h@Xy}w3EWqEz?MrOW`#u~|Fzh7Q^;>h0J+)pcvg(rO7?}E?-lAfJzdDQ1# zHw90A>pbn~jy-BxB2u0Wpi~Pq8x>D~f`iKIH$VLjC=Y9R?sz9#;N{Nbp+|juyG|!I z_NYO}m_c60cpHN_~4~sWQY+8*} z=5_P+5dTZ`69rCLeQDV_AX2Lz=BX0 zB^ENbz_V<;M1v6(b{GXzimsNn&-DVvljFI0u3~?>zoB5$@y@I3$jT2ZMw<#@pR3?9(!#J?9NUJnvV9JIf zNlKZ8=uPC;{aLneTx)A=tElT`@t5vQ4RI|>_+T-5xy;EQ6sr74Cl&;5I~}}Oa0(i7 zs@+Ji#gZkY#-R1M5eWSYc9OJ?55ctSFTrqty0=b#mBptZGuPQG?mA3ePi6AJPW6Ew zLvweNPODE=2l5#NfK`S_MM_Tr8CC+ksT4?UquC2~8V+5L+%Gx5 z5gJz{w&17sVC{x6C!|)*NyNq~5*WA2eZw#Xa1AcUstGn%l9(h0}6TMSpbhjMfI6NE8@X;zoE~KLqLl<1PK$XhUd4OENCY z?y|(lFhJTz%|0;_nb=@-g6oK*HD7nyS?C`$n0fLHj|ebWVl?H1%3uN}$|)tK)rBu*Eeb=J3N{NhC$vC%oj*%*pNROlv7O z=dz(lmtmEgdPITV8SGu~?0mPukq{{fHOAOWgsRB_vkT^wo>+v_yCrurOvrC|2{NCX z7*xNVZUZX$n$pAH>hu|s3D92ICYl4wySO4x?z}|zp1-_YsLb9hm$IxyDlsXnIjmrT zmgp$Bxfl0|FMH_TeO%&nsPx~MC(xH1F1}1}0Smr^KFoo$3-+dJJmLH=D*r;fV?h|Q zbjRq_h8?S;xBLm{TG{1O$=lh&I=D*%FAeut>u#Y@P##hlhZ?I| zi6=H`%1HH}!3z`WjAWDppyd#t#_Q%0fvzQ~;6O3z^AU(|`PnENT2)D9oA~2I2Crqb ztY`^rVH~nrKFcx89;_Ov8Rb&8PJFxQW=GqwgIyZT$g=Pt^#;VC0B@r(( zLe7x{uV;KF*X`zg6R=--76lKA;ebR|Q(TgK_O{tS%B9kXN~!X?36|RwT?6$b6w2|O z`aREmpZ9Ec#|-*Tk>!4s=;3>!!(Z%$Q0(g3w02bOez;7L+Ti|+<+p+VKq`wNj!UzVvDuQ#OSHR@?Z+^J*CFeh$02L%M@g+{kB1L3+8=(t zE=1SA&!&B@V58TLI>$_Z`C1xJ<0;Eg)b(Vd@y$EiSX206kftytqn*P!nc=XY^PWI$ z*s{RI=ebB&JVnvF9I9q?Gv!E7z9SPznheeyn>)bGU7}^VQc(-fWah2>?s0=IVvYgw zn+BVhuN3xcU5=|||FE=KJ!Q3=ZJiqihAKna%zi=O%y47kZ+}mP#_u35W|7%QkKu=L zmIe72Ls;FJ=8?c(^m4D?&%Qy%US^r1Qtd3Y?KjwO;uNURX@M;&#!FlcGZ+&~5{JUQ z^~KP^QAM5{N{%z8R0KO~ll?X8zCd1#_>?g@<+&h8~ao{&39P&UQd(tX6B zq?F{2EN{1tI~L=~Ulk-Zqnue9E&G;c;4vNJ7+Za{UCJFwkYE`cAu5qFN_TS`QP81W zMs9oKkYxr|rsn$PG?Nk6w#OXiA1{V%3FY;)2TB3^h=X>ha99v@nhwZeul%>{{*=P^ zH$Oy#0g)V64dTkzXVwkcG9m2m0fl8liM=)denI-jr6P& zV<7Md7C!>4sT`#w?EBP@j1`MBr>hAK0FitpWz zyi%^8?|x|N-{;vdwmj`vO|C4vGkXl zt6+&bRDwVcm0ej*iOy3P+MpcTCr(4i3JswNf?51^KFr_^;wtR%^>n z{MvdXooNH-M=jrVJf3jrE(V}~7^YNDFdTerjzOA&rgeLk81F%SxG#;v8aHRlJS5iJ z#U)cRDJm+0Q28a~}TL4zo<8b0P!pf_a;-`EL#ZWjPD zwCNKW32q9m+YPHSowmA;DsQPJ2BA0a7`!GD#*ZXkqFIseSSeF&16=#2*QCgAw!!Cz zGk2}|s0$MLXbMXCXbn#Nbp~HdO>Bcsg|}5RyU+nQnWOJYs_E+*h>};zgWeXD2B&M( z*R2^Nm~5(q)s6Z^P@U^i|6qAQ$Vj^yMPQm|tBt|uEkOQ&!_y}kKk$hD1<^)PqR0`s zUdpQE9o^`%d1S$Qs@+EoWD8KmaQ#Kg_FDvIpA+2N9-Kn@F@}bGrWlctAoo;ISwXh9 zpv?+VhAsSq1h)V^RSrY4a|iWJ6^3Mk&Kb@jF=vP^G{Jm@}_1 zTlKE!6x(U}{s|!ZX{{2Itzyz#u9Auc)*R-&+W=K)Yl*5lcUWA`2P;oM4T#gg;#5+M zq-Rod*`&uoVu{yowN@~B1QTPiT;P4WVbZ(Q?kK@qX{eX)CZS z^EKzfKHPsBIwBcMo5wy%mx+*D7hY9Lvb>!=JKZe|9e#h@x_<7B z9q#zPyx)W*NjeI1-t_~<2eB@|6~;Czr=D~&4~s)QQ>B(M%N#wh9e;DE9Mu=@4><~G ze_1v_`VE|vYm|p+xK+BJSwq6^;6|C&jN2cd6Pa3T6Tn+%q?Gt)3_TpsuR`9tn|0Hp zP|drS9kjHYgw)b5u1FxNB3l!VgKJ(zGhxdhmz8g0=v+wYl4%Wc`juf|x!5!W5BK_k>)W@ zt!RUjTPAL)8ylsMrL}QLtB}}OQpQp|B5s)$mnu#~Y?f)MtwT{YJhM|y;lZU^ldUeU zuB7H}DD|azYLb2|{`Z*rrFmacrkV$YMybMx|K}R1Sd@k6k;O>B2qAWFU|SP!Ak-DR zXXJLT&lQJUyYJ9NcP`=0+7i}kLHUp;HtQ)GpR-jMTjUp{GUcLel|ivDzaao*g4zoO z9M|{$x;Zsg7?#cNZm3iNZ)YF0mAZ`65Ya!EF3Apec0beR6ob}QmGEy#d?Sh>LXeBM z;_yfr?#@2ZjgIZ)J?o85rFteKtSC;J86>&tiJvW#3YTb8;>n?TUFA07Z-_>$MCo1d zn=!6>qVwi;cNL4ht>uH3hY`xv*7)dfFQDgWZo)XkGUa)?K)$?8*DM1?OXlzLl+)-r zG%I}&+T`KWJUdvpK;Dcglw#4`JdrVu6(`U7`b{4XlRN}tBk<7;7CUq<^Q%V$*GhhV zJ`2p_fn>U;^@weHa-ZZnlnce}#)QoR8*h?vi`6QuYgG!KD(s{u4mp@ zkCWZ5;pox-8?)501gTvQxPmmS*PbwKUjH47EC2NypLqP&-0b{u;|bDmx5CV@D*Z%f zWFl0=y?Udy=~>85U#jQhKZzh(M>R3vxOSm?{a7~ZrWr{|)FU;pLDTA^cRPGv2=rip z9VHAW!o@C^#{c1~a}Ctn1|pv<+>q*V-hQzSs`dw4_f6Xms7fuyyQq3I=HEvoXR0rT zkCLc%PK-T0ryrR_O#)}?+p;ev+vMp$|7qd&s!Cz9)dXs`o+{FYAdNqPa~=1)qFiq_ z!kx{MWz+Adw}R);yL()@-wVbh+LSeTkb^y4616|tQMzUKM^lY15KE|CCG;CS0m54@y zQUR+BswVH>Yo7Nv0PDZ{O}^8*{)LBJ;#(C7Sw*S;Sa%L7S`wBc(YYnXdw9f21>(yn zo78|HfFx_Ya=t%tw174ve)TGh)sqS3F4Pm>Ax#Wzq1z#5ZdZxCNVb)jl0sTHe1B0D zy0J#M5`F#|;$(_&-7rMC>6FA7Q}m45*Y@@@YN9wtP7+T*3DDY^`+%sM5Qg@gu|6`l z?5&3Ow(ALG(B9nMiJ$6S9JGGZXnupxU@2A45<}+?9621If4$XzXC=jZ^0|mlfREnT zTZ)rG*x7J+A9mH>e*-8Pw-$_yQL8Y0ua>Fm^8;3AsA_7W$+ ziFH`>HtA{aGyD5=+ulc!Dk(1wc*Ckiww&z|C50d=G5*-IzFM=er5oqe z7!wrLzZG|tN}g$L5E;k4e~h`qnb_&#?tI;EgmK+EzhV<;%n{<%DG{<(B(O!RI1z$* zq3T1Gbwgl;|3NXV^o*`O%gDw?eaRxNG2X4zoOPccC=9P+DdJS4&3(#KO4zVHocM|% z05NGG0h^)NpgbY~7?sW2xaluR9-CGW^T5qGs|D(IMA`GQE*$t40uQ#?LF6@U zZ+p%M_;L6t75&_HJ=8{nDfuP~RSa_v?Q4B#`)fG`yagL8?D{Q@@38Kb(Nh$jG=XT;=b>@8@tTTQrx;)3T#IQu$`UVLib4}cHOhzg=8yRwZ7{)$z z8Olo>L5wBH@GgAy%{bFw{(iFiz27`IX!#KNAYZMWAikx9KI`qn*29rb(39sf7053TGfo3qKuneyJEC-jY{^sUaj;a-3)` zz5lJhi^KIubl7lJaqGGV4cVRO!Bns&1{iL1j(V-n-lI-4AAjRX=KzO)VIQSIu?wGw zDYT5f-S&KiSW|02)a)sQKTuRPm%tKMESE9rk#h6BdPQpHxLy5grFpFMI29swLI$C`L2b{d#buS~0BPFM&3qayTgeduh$h`uGhO--LIpu zMB16JyCb~moR7z`wG&@HQ_oC)QxpaLf1!M-!3Pome9;4Gzg) z-MKVn8fc9INcszWrN4ETDC4F6_}_eNy;;4j%#@mz9#X5dx96l26LM>(A1^0Sm;6Q# zlKf28wMcs&Gh%oxa>(QoW-C3cCWJ|weYm#Wol)m~J&b91pzu%;8&b6m6A$yQ5vQ=b zl+_}^leWuq>5gxJ{jAsi%ScEH-CvS1Is=taECgiteD=8ZFov!|;aZu-GSQqglkwmSN$S9 zqZDgdtV1nfy&3a4IDRWS1_E-$pur2yg4M2#f*22xK-+X9sF4{_177weU`|(Zbsja4 zX~<85jzBqDU-M~ikW|Yy4qwliRt+|1p^W2C33SEgmBfpMGF=5rsh#|3)$IC_xx_Lt zXR;H0AECN}Fp@H)LxdxTe>r1feY*rN>J!iS0Nj|Kq!4)0W0s$YuH{47S@4nwC+lZs z*3V^-=v+Kr!*-G3DE8V`c0#|9eZ z^569pP0G$QHrBQS-J3}_@6}S#D&)ZCADv%+6CTr-BEjL9WJv>x-ni^?E!Lno=Q!t= z@P{m%Cg;?N&N-dSWl@&^8q&Q8lVNUWb!*d>O&iRRU{hW#*#vuJh$_&&qhq01Rt+a~ ziDBMQp(&RbO8dN5a%|#kaf7yY=d7$?*5ZPoBD*tt9lp1nq{>XT6n=CnaSgcg0VL^i zq|JO)rx_30A-NcDXSccePkPnv>csPVum!X+G>BJ2|Xm|MBdUl+vu(ZQ5vDY zu5+7>vjwDA9QJf}v$r5o@m;4`My?90Xfru58ArN!3$y+x=pB`&v8UYSMB*xKQes)* zKCw<#5F&Fo?f9P)eryTp7kv;Nxf&LIT+t`pp({`sd9N0Z+b#lbUr`QZ@>TB z?uqrhN%Z5ZY2R}VdL$usDkdP0QR$@XkHu$ew$r)^^Uz8r^Lr_2kBW*@TE~)9X$->2(F|PE_Y2`oU?{Re*9Rm}C@8K3j5YhTmAxrM^!Xz|^ zIv}m0^J1=L2f6(Y&J}w=4zt;>W6iCrBt!#Tr#{gKO(--kXGI5hMa~Jc?U@79Gin@F zR;VebKGdnRRA(8hw^_Ykf1!~~aJfCv@?6ODtrnnt?=vnC085IaPZ}8j%B)f?=r#au z8&}1iCibFYUs|_ZWGp#67`t9aumlj4`tUr=c5_vc&Py3uOkCft#1N5t5mkUysX0sy z#7X=MU>?(hFiwIeQl^h)#u&F}Am=m>sWhP#6J>1UdlU{LJPtH7Z;{=stc;4)5NJ6S z-9|_KwUo_XV9Yi$oR3R?&b)ssgNrxq=$M__8_>P?Zy`?VB9~}a00M0UJnKlGUiJDL zZU&bohs1Uxth;)v1Oi=KuTMuT?1zT0PKteeOK&^3^kN1gN1~j+VV6&6gT;t27gGi? zWr?n&KL?>B%3!`fuGSx#s5e=9qWwq9kW5v`dK zt+5fUff22)5@nVWWx5h|iV}6A5&%vK0IdXoNCJQ^0f3eOe2W9_#Q`VcfDLiLj5uIG z{HJ(rmI>w5w84-J8NN-_7?@Q=|oI`xyjR36fXqg=4(G#Jne+ZB81`0vVIxGS zEQSxCZA>ryl^QGvzt?Gt|)PNd7mJ4s0uaM=bZ0jpYKOGi-%KNzx>;JV5T%vx#fp! zdC58wtl@*Z%Czyexn%YFH5Cb&$u|Nr&{E82m4CH>bEM+}|GqV$rKF7GS=X%m^Oq9> z?CVB8cXXfaJ&BmC>RQo;MN^5a8bGDycR_%vM8rrvP~5;H)S8GLD@RJ@u(K~=1Y6^d zC0}YQCfo;jQ`2%nJbG{c5xTQx3$fi8{#4Y0k zszOBZI%4AlL%;9GH8^NFM0KPd0RYlSy!eESq9QFPKZwF21&tBDQPBW1LicXPEO54& zJ3ZBtlEY3A+2oF&tOa4G(iI@06#C_o!%XsS8qg0z;*T~&Mzh1jp*Y(aqM1v#f4Lcb z8H!gwjKim#tWykTlDbHn?C$w$rGIyScZG#Rb9OSr_qf+PtL*f+=Rf<(Br!Ae{m?ml zrPVuB&#TQaw5nIDw4rXjZJz&tY(~SDN1on!eqom06GgUh#dHwYiunkIL z3p$xgEaKOaDH`_V$SD@|tP!xYU3(`-weoWmr1F1d))WycxyI){sn@K}5vfr>>JW2* z)?6A4z__;L7}{BGVa2s_3P&FY^lS}o1+1jpi$p2BoUJux;BeqG?HXUpZ4%mb3`F+%(eO z%+DsWoA+r;C)-^w^JOvz`A70fqTwpRsOru3dGu_m-|}U^zspSld@<6t#*T*} zq^50!7nw-f0Ak`LDa)09N@MVCXWQtj>8UpnU3X*$`G7d)0|Uzg+;f>et1!+ZOT(;& zo?lDz>O*!ZOQe73axH`Vt3v%UaUuxyiO^O-wgb$d;s6?Ql^Cj$036ss(k03Q;p(?w zYH}SC<^G;^G+o->FsLrj^lL? zz0`_tkzhS$<@H0puxaeuP#C+%Wlu<6@4Y=RtJHgAuw0sk=AxoifH@0e22{Qn}h zac5sd&iwmbQ!O!sm2iPPFo%brcO&v30`BOyxyp`C4%koA%HO+)7{xi)TF?7az*a$6 z%6W(s+*-{Q_>xtCKWjG~k1hZ9M#C<~A6J%AjFKIR0_<$ol7QV|O5H}E=jaus4?1Mb z*j6v8qG%`1a$%l&S>o2>fs}iM0=^y(_UIFCuaso*^6{>T);=d6=2>aJms4G4-v!M{ z##R)nzl)EB0|)WU;6xO0ceT1fCq;QaG;8Sb2b`eA17GBU!-;6QDqx@Et#Iqd|F*Rma)J{o!wi_*Y<0Rw-Ohyt<1{)RnESDQ8)B@OW;Z>I;O8Z2^fdf<9^ zQi=d?U3Z*bRbzlP)_xn9#txB(&3R);*A9`V_4!2E2uZ}7k@E~eOVF>8n7lYjZ8G|o z_)VRGrEy2-h}P8>VhaXLXDWuRzT&)dgY(?9^TS=gV`!*!Q5ji%j{r@^T*t+GvYX`1)*+;4#sDTwj&&eQG z=5j7v8@mv>ThEXxp9lP~#27}k=xYnq#~F5jJvbB^H#i+wPF(ymfOdk3QO7svAQYv>hD z7GH3etjA73afxg^QZTz#IiMHX!1q3M1ZxZlnnV^kzW-agLXW%Cuc!`I#~A)&ct`FOr3Gd0cqejUTh`2nZ2-*<;&siwZKckrgUyzX}^WdFgX8ilU^ zKU}Jy=YLaLvZX-~UdMB#a?%_Qz4yP6v<6sng)vd6sFZ5DZ2v7o8kq(%^^K~>$zYo- zaqKWLf`xa57$! z+Bl3&eg-Kxa7~1z{e{f5z+$E{OF;te-6~!AiQ?$4aunEbFPL;%zE-V;%l>T?(zLwT z43K3CpH&2LJycx$jH^aAD~&-_x04^oMqaExyG3v0Pgec%+PmXtpj>j*_VwnkKFl-? z?p=FgfXUZz$hUa>H;H2RTSoz$A4;><*WewGOSo1mnQN+9Yk>>adOobYR0{E5l<81J z^3Hlf+U14DX<0D7(gOdfZbBnfRf_DKlI_c4xBj;U+-2zSO$(dgh->A19!31(FH^p_ z^vhSPi<#q&nW>KGn`OcNPCT?Bs4Br(DY)0H+Kg_6{%+kpu(W%VS@9qLDht0o zD|7|E-+c=QScN3*n265y{>?n?EthVBe!l4r7Zl< zI(??I#V4TAD}2gO&adae`aY}fzH$Pu8({O%zD1uePH&-XvXKg3Og!@f9w02Cwso!$ zz0E3AU1E_>W_shny*9ROlCb`WaW!ygkHyJ)=_1*C;3$R+;cgXC!+)KW(S)jA@j8$J z33k$~_ppNWLmO2=S!+N7YBN|B$<0}Hqw(f{RF&(qxYhEr2sLTS8~@i9RI373XyFL; zH*y+kD40unEW@ZYfreC#Sh7N_0E%D6UWSa$*`!}b<*XF$@lZb$ri0`aWa2i<&jgD_ zz2~Rs$+~HwXh%zHM@g*~zOd`P?xwlw=KnzM&`LqerE2Y45ou*x;N&5z;%3r~D{zLb zxyA-e@%W3544F9+!I|3#qnWx(8QzT(;M43B_jLKwA z8)hQO&Jr!a&hv3nul4l?`ASvm{Jn*zitT*I!Pgmy44oud2nx${-Oh`GIO0hGm2`FT zEY|?^SwJQjJ@e+p6x!rkiJr7%CV@PTTy(WS0jxY}YKuljNRDCoD^d!w#(XE1 zKW*v~a9hNQ6jQ8#k_5RNkXj4yX%gmplD?=g)>lU}c(@j1oWQDy@?D(ePw&ECC>t zs9eL0M&=+`&1wwGGeL6KPfb_+3krM02G=*M;Arh^?AzAHJVZOcNLPhtT`Wa)PlrOn~6bsuKhN=>dy1@`@{03@hPyuT5xp` zO2WlT)0Y#{*C%y2EkWOWU#h-4^ssgIopobR2g(~7n?euG2+kMyHpj3JkNp&Wri>+aIjmO>%)9$#BVZE0(0uh13{9s9+5bo1!I z-8-=4-WE?Vhqzr0#C>_9gjXPH8M|H8+x^m|*5O&#=|kBQJHAq1Gr@Xvq#@{~;06bN z1h;6@-fVdsMwK%@a3@@cT?B@rh$w7wxSQ|?URp?xi{zwB@jMO9Nsj@$8`;G#hx_ZPQ&Zfrq}uLIHZ|?@)ZG#GxY?7QtwdziXeyd4#90L&Em#&RhK(@uLpmml|urkc97b&g_uF7_v~Qk%g0T)?d&K z^O6QaA^$}cq&KYLB}*0@cA|KQD&~`{B&ED)$!v(@2_7QKblhD0DIVxzS>al@mN}@T z-sLb7xB5Eu`Sfh7JeFuqH^l%3Fp}F|t9^fDsK-z%8~V$XDblC@B`s|X0tmepNvF?R zq|JCPn%D(U2XS;s*XbHdOJqmfIL2kAxHM&diBV@MIiV^-7a&Zb2JT)xZ>MBaru^}J zat=XJzak|Z3BXSg72bQ`{WKz#MEhQPT1wMq>mHF#SxvUuC`)>+Kc+Z_MEBb3V4~4HHjIJX?bDhkfHNOo}!)1d0vIj zUh9u`h4=;QuOXHDA^}#5)82g6T8trj0hsa54pXbs0p5D)KQ`(Fv!R}|3{#z54Q4eZ z9{i6!>ItqYrz_B4#dr<%jmP}o+{kWaGWp+IOKynDPAfEKFZzS%yUc}C=N=I^1(Lp> zEz6w;61Q;tM z7!4>dn0(7~xxLVj1Vjga7w-{-+RzX`HVRCP)O6{KIXM z5&~$OqOHicv8CFyru)7nnFT+H!}JpaZ83V+9-zOKoEjujKx&RIs)+ssp?+Bb26bZ=KglA2bOg^Qw>A&WD)uxNLZlHM_d9rdsz*1?>3 z|NWNC=m63N7UwXgI#^2q*oF8_Ul0^VEWyLV71A?bTCF=r?k*DqUP8 zk05pXrrPB&J6Z1>iNcDmK`Y09?-GDtVVee2hw3^-j=B_$w|0H(P$pUb7*7!@0U=!~t=V{C;# z8-ioK!NGz+`)HQ7usuQm%N&>ggs9FL)%~G)+~6Ns8)d1JxCW=Hx1qL3mkArRO5cOK zu-08-F=fj7<=0pNH~+Kqb?Ef{>Ga27uk17grA!;n8 zal6@)cc8Cj%`jU}r|{kZp^faS#<|!V8&~1z%6ptOm}iF`tIlg;vtGFB#o-Hl*SE)t zoD_O54D2vxQ*tket4`DvGD>rapwbH&&s7ke;|D0sslNvwHKW+kkIYYe{tc^{+BAw5 zhs|c(^pC;T-#(w>yX^M;D5p8CwCCeb)cr_X0a8=5h(NsE)h$%hqIFGM1CM(MQ(M(w z;x3mie+F}qDKgUl(TjORU^-pdqG=Q!S{gznkao~0KfUS%jZI?BvIHx+5nU;ZGbhyw zqiZq~*l8L7u*{}a=0(UhRq9*kJdQLbCBJI+O~&hoqcK${o|fi|s{N4_opQu+MIANk z5*k{Xx+`a;8i6@%auo1_nT*}cDfmoZkyfnoQyhkIc=X-CMYfK;Xw|2Z6jeC1^iaty z3{Nm=n_!JD1_qPBv!~0HKz}nQ!xt}ELlU+5I5|uKEEF!u<6Kk^C|ppKX7Zy@JZVup zF9Nuu%xwL!&s?5g;?7tcaRQ&t*yvT{k+sivM{txeeId=|nFB4-OTi$4dtwWHmpjz! zG%7_gXv?!rSRi9(%NH_QIEe(0oJIxEa}FQjBPtxM2aVY6;(F@Fw2mELb|YZtKoI$B9PU^*;O!I@1f1W2n0(7nJUiGZHGYHr- z=AUn?$K4T-zfm@iyY~Jv7GH27WXtyvBf30^I2x)AFWxusdmrVeo{BN9TN!$<#JstS z&#*kR9Ct9=CzP2hXf@mtxnH7lREhl|>$Zv{)mhaxm=G@l?1E`UAR!v`_pG>W`?}?e zKtk4u7l-A2lm}@B_@At)DZb@(R*#IXd1QH2YdbV=TVf7+w+(^=2KKFiJ5nLfU2UV8 z16&?3pE$}2Ec{gt_@~RCbty*=;@6T6Rc=dzmm0>O{$dUz_`x%P_LpM!Bo;Y>j{+8#C5vEG5K~yuhk;AGtff;h zkJ0fsoAFR^)R*yi%j1qf!62mh6OWYIwDC6=+69RVb)+loVi8+tX` z;Uy?Ccff97=tBHt?0_9GcEOCI`!#FI&;iRee8OZAJ!?9HRXDNZA2mV1w`^>XBQA^@ zsf_)kgG#7?luzwWEMO|Y`*#O25RYajq8W@Ky|Bdrq&{V|-2 znBUhEn{1S6-~l7EQ4ygD)?SE0zYy07)6ta3iM8d9AR9M^wZlmj8G)6EYW21*ghWCt(Z|mC_(eTePzlCqZ z1^4$jDjzj80%A$~Y&_Cg(KxW9wPAmY zG4*}A4;X-rq3Ca6=eE9YbLm@)2ETe6SwT7eZF|Dt=KONf!TFzqoNot$#`0P55i$Qd z{1&bLv)Fy#5{|gS34{N+FCX7`2MiBi7ef2LOZ4^6)4lf=z8#xigfl-4mJg3CqKw>& zXDLe#W`vn%O`aNg^F~X%dic3|>UwWnR1iO56MxdR7b~|ZNH-B#t!8f&1^*%;oGTmK z1};?!l5p{J4gXUlV9p>|rXFamP{d#qn>19BxI$eM53X3($L%B4AVfu!LsSvP%U{ey zNgK5IT$IDCZ)q3YSY=)M*a2hN;iPl>aHCLQeoQetx1Dw74JrQz3T+1RIEI0x6jM+m zf1VYsw7{S=fh3T^Q;YGK^03qkf`yPn(xZRj^ugi$+z&Kl%a-n>wv9i&&@lz__z^vh zMDn+`B7aaWh8G0cBi=)kob*I-zi{43L_nxn2&wEtsh#{x^=c}1H*G62Bz z*w&A)gk%UxQ!3ud7J6m6zu616(iGvFqd84TW+#)IEkXT={6;WOKo6f_uWFIcN zc@9iQQ43es$`ED)%tR%RiF3cYa*!Tc|Hkb1$?st@xG3C7c4{L?ioH@HX-;pF#ppW; zPVj}(G73_0oJ5X3D7C1PCrV4J&y(*&a>%ksN!0IDwm(@(pG%*0HWXXZEOOWwe<iPK-s`@M|k9&L$MP=rp^0Kn~chBtx zMK0*m)moe-tyG`MBbrvy@?2;_08J%@ODFfsy^@aLdsn-p`I)$M$vH%sNj=p4V=TSGR8S?0B#Wtq6>v+J~GE?P! zsoP<_4z`K2_OQrr9I_$>e0j*|ee8VHxG;s~PJ>$vZ~bMQ_+~{2d3R(A4cC$WB0Pq} zffyI_bW}o|PmK#XVVA0$TrA>!pTBOVXy1~TU?MGkmCz%KP|8gV*+y2-mxz7EKPoHg zKFfd*$&_LY3JpbctAetF2{#cSxmrNGEcGX;g9|rqM=7tv zrri6qYpJt_)2ecIY5nVVvN~iFTvkb-gr!h|Yyk6(Bplp}UTuwor)m$+@vujX8Mbo8 ziUMN5KLjg9OHD2mq%e4sqoXq`n+$fZkGSdh_R5lnM*(T{o@a_J!{X$mbP zq1k-p-MI3TnOI#={-_QH5h>eBZC#Rs--O5r%hfY>Z>U&fQ-3`g8jG_2OT-e;pXX?BEkz`T0i1Qj2Vt&Z17rU$&QbItsHYVw8J|z0+F9< zscTTcPx-otiPN5f?wc1fbG0SJE?`Di-5;y^XiPk}oF&}i_dlh;cYMw=TzkEaOqs_*h!6)eEQHe3UQ1!m=0CnDS?Fjus32&(fm{|)&O zN!@Q=^!KKYVDe7*X_&ky16RQyyv>kSAVyX{`09i7bH-4QDerg0^hyg<8|$%fIFz(3 zdKi;NxpmFDV>+8fAImR+(BDepeT#k`8>C-GO#_&+6!a!z&y2!vy>#fItHa?P(}{CF}y)~EeH9N+G5)o-TxadCa0uMRuyw>P^w zJ=|Sodbz&N&iALi+)5JHl9whBn<+23s-^fIw;QQ;22)$3OyAeigt_a-qMgEJX~v|s z3ByQNJ1lVx@~uHS`o=!pf|jp`Y- zpWcu1pRIq}SQWpTy8{A!&E9Yw3?mtcBLB*NjAT z279=w$-2caEiACBXZBeZwoX!wt3$8Q))-Y49?+orv?3Qr0HB$ZiM)2$q$cI{P$;6Q9BRv0 zDzKI9Y-d>#D5&d7EDZt3DOKGRxv9%X4{}PrSOsfx^*_1ENU?B#E~t5=?c7n0lX(!O zl1#k)!TVW3QZE^w(*WZ}=a~9c(6tY-_nnLwZ+U7IjrsZ3m+|)aVs{rnr?=Vldc1f( zq0TO7H3Q%hAT=rzQ9t~5_odzVa zgy?0PjW4ds6>L0T4n>a37P;8Bx zCrv%OtN36%K0Q?Zv9Wf~)TcFT5U8gZhFeqQ+Ze?om0}XSo4aiR);55YUGOIlc@t2P z%8tIDQ@D)fd79Aq!5*b(P1RC%MdV zlv{E0Me+MOY3Go5xbN;t;QcR4#FcC636Y9FR<)>qWXs)LGByPR=_+HcFM&5so`*>X z$V3~bL77I-1il&FX9IsyaRsqiO`zNfSr+?E4Z*<3BM`)X=j2GsdY-15J$@&CRRlgN zhBSU0&&vCIv2%W%os2>>e6X*{V~sql@!m!x@%7?${xe6GFX3L78`ITtP!6% z6Z$(`#$&sO#cj3B3}?9v4(?>?hHvrXlSC_jGqHNcf+q~K*Lh!%n1avAw(xK#KQ~+w zY(@g-2H2|?c^C8nGllyzKyE=D=a7e-)7Pi4Sz<>9r^ zv1SMbGZ+exy6UXWq?wIRkaaFIA{83Tr-%wsd7VNM?KqH^FM1V922&Filc=-&gA`?J za&{KtKiHwp0Kj^S2vw_3TK9nar+~8{asqM+;t7jJoE5$7XCt76t&UL&u;264Dni%0 z8t|!cr5zozWT|++2l7mltc-IKwE-3Am$%*WAOaSgK|7_tYLtoNFVC02NZn!XGpad@ zDi=hdiF?x1n2+D>Bt$owI#xAmz)Y*g*RJGf-Z(FX&Q6J4>9hj5IvyZ(#m00{JQt4N#iTG?NrY@SIdmBa8Ef#-dsXKsqD4LRwqnBB* z3_CjM=@!d=2i8qwO(ZvS_<^dfE_QZKKK;0Z1qw9F=>-F~gsS8LJlF1wFIZW_wWHT2 zV18A8@xvZJN+%bDL>6^&nj2qG?3iSgU8G{X(9z=*ImC9LT)j508Ap1^ znnG)8_(Z`ezzXsAtP|a0vs#l6NlUn``V}ger@5w>l88EWg4Gb!IsuQm2my|&I08<6 zH~|E>&B!gOnWY$5hH-}GV|6|T6_Hxf?v>1MD~w(gi+C*rS!6l_eJRJVu{!?>JJ`Ai zsEWXfl+?v0Gfep_5~{R$Y-JE|;uNhRwc;P#c0#JoKRe4&y>LbU?{F0pHp-1KLl#vPf-SWHs~RNDx# zq{T_L6q~#dK?Cag{3IBQ&Q>a{AJiI(xOgAFwH}T~*Kt*H4naS7$ zr&OWvCM29P@`$SSn^&x&70pXco;2hVPDNmJiF&)OabEN=!W@Np__#gXJc1&mBPThH z!r<$iqU3r7^>Vt<~|bP zdBhZoLTjfE5)NhQeOU7>SwGUikt^H&i*?4Buu-!=7HzCNJ=%Uv6jkM$t@+3S)#q=| znVFUL;a2geaATsF0b0_7S&@r;r_%nh;M+FsDrAJK4z4AxibL2_2Nq0m_dj&?WM}z3 z^Ue8wH`*v4HA@xD{QTMwJgIW9x{W8 zqQ*jQR*Gh(yV8r>(2)hWPp4V|)F;m-vfRr_3*QEaa0MgO^LPKyNE8YbUE8Lyqru(D z7tp;&YGy8;Mm>4#ca(?-gZtYAVw$t`V}2$Vm5p*X8shfp2v@B~p2*b-d9#w542c4yzi zM35anT@UjI&2i;XWO|3*|LY#zKPSVIOCl}*8&r4PtMm5;Hlff+?y$qk6gOi@ z-lJL%b#)w9?KL|3=lq=<^So5yW6mvSN|_9h;1j9)cvPIg>)TVw8T%UD{6h1M>u@d< z?jj~^l_Hc`pqa@OxF9o#xJm~56J@tlf

n-z?(6G*_E(LEu?>?5F%6UQo0wf!uCtE+qroys z&;6O({h9l@>*{~H&U~z{t0&;EZS;8juB)rKhRK;VGYg9!1MkuQ#oj%}*b=t=f^WNf z_io#^ZQHhOTf1$$d$(=dwr$(y^z)qaPBJ<3X}(Rezf|R}TB%AUwN`4?ef_TgwQR@F z&W4|Z{b$>M5Arkq+4XaF>gk6e=E8>hA#JHrc+3x7(~dm%R<(znP?FG_#Voy`HmH@R zUtuMNB_ea)Pw+EafJ;RO0@UpML`X||Cm)yPP>OBLzN-BNuDdlrlh6(Ek zj2Akj=qaDXm=hvIloM$7W2nmTHY}up_&V_Ca{XcoO%o7(61zr=(iB}G0Pgj@&IKi( zQAz_tb25TpUmfh|YIMP~BeSmHqZXThQ>nBZ|_&T6N%uaetwtB3YY+Nuv| z657W>YpEk_nox>jLGh9CIPV>4xuu2R+tp>bjO_;IDH95@1JK0swy*vkKu7arg>x37 zcWC;Dp}1Hb>#l5XS1pMBB@PSkrm7~34vy{0Iaq~kG-C{Trr|u_zHW(X^(1a?TM;%P0JyGC-KZ{`oca48yr3a0g{7D3MJ#xk6$ARSQQ++?J+}(0n;oM-A?je$&_?U|_T62v2QIPuT@pevE z5@mabKJbWt)A^RGv^YP&!?yDgOC;+lOUA`^iJ+Q6`YWmQe0`j5e}^AoPHJ=FTja{y z2i033GWI3EMfxr9eV&M%nlS>Jea?0RPG@gVWaRaWODvIYwMsK|Wi~zdw7J&qpddw6 zBVSv9<|219KW=HzQDx$*{mfoQF1{oUS*_G6#BXwCExj)^=fgdqJl57a*@SNlcxa&OYE)N3E8DEgB{*88 ziP2))(4=q7YG50Op1fipk9^y({%(S(XnuU1pZ+$fg4_f8yd)pUA2Sq;^((RnPTQxN z*HD=mDPzO(!Zg^%tX9T?5{*b)UgkAxt<9jI5U(4o9gY0~W59X>;K^t0p2!S!`UN&?6Q?n4QfiEF%NTO72`_YK^xjSr?P5T7IV@^Vc~w|79> z>8DR;t^k7joCh@5mF0zi8^dFoOUm5`O`U#!rS+b2FqBHBc1o7a+#b5Gn7*cXlP;iFhSxtly;XdFLcbmu`U*}$Ub5{epNF_>Fr1db6OS89n`?$dt+i zHUNNx0@`rqy4dm}=}I|3u4V_n+D&|+q9K0C1hZ`uiT5aasRub&4okar0DdyVgVo!` zsV%>TK8t3-V;w=|(bk2rPF3D0-HtTm1jZtum+{m!)%;J z<(X(T8@5{2iT3iy3XN_NSJMeu>W zQ;d7h#h?GJ8QLL)JM>OlP-*{CXdBY}qUeYb220I}CUmi!IR<8$trVXophbR|gW@A` z6<3jH+XD9vjmxEoe!~oxV=2BRz8pAv5_to6V8yO0{Ap>pur753UN^>eo0YGk&hpxs z1uxB%K)KgQXva77^|f%+b$`j#xOIQVQ<^B@4L$9qlbVya8Dm3vHc^hxjmWi}?rB~( ztgRLAWT)Ab6GNIR@Lp^#vwC_#K$k>%eAD2xOQSOPROBRiBzF^>JHHC*bp7@9uz5Im zsA|pWdBqA{y;Y~9llR)yhew2%ED#6bUcsA&aFUd45VTIGP$iLcr+Fkj{sQelGqUha zmV`4X-56OQ?g2d_lz>f|o>Wz2*jB_siBnv34$F!HQ#Z-B2eTXEVDeWqh*!!sMzk+|;@m9fZ6_nGC8Fq`XM=BY0 z4^p2ROhFvTB}4N7I_9@`V#UX$qZ@A>+?^Y(WOT{pw#G1z0(LMfi}jmo`MH!T-5{Y* zjr?44Anp=S^J@ldwrTC~NZZY3OL?CWPHh4#rf#rDTM4!;)84b}EgNroR4VT5M#t0Z zwAo+^C=i-`nvx<;$nglDQ3EKmZz}jf%HdJV58KSNC*ako%;t8%7SnNb@y3#cfcO*H zXc-JRRPo9p%{hAuZ{h9bQR~KEE(;X#X^H(|$RS~$viiFVpq>UF0?8RN*)Xh0d(da7 z(jc@N?iF*A3jJ;vZ9RKg{>aOZEgD5X(i{sxrkklG8xPdf*>Fz!%+!kyOr)V}Tkl1; zAFlJ#cM2!DuWK&weuG#3mF0c6dTStNZ~6;D(;_DJMl8#!`oIg~K@(z!QUI8d=(z#c z_e)D;JaXDz^97rYSqmtk;2ZEwiGf-Xf`Is>v2I5Sf$8{kC^o%pizN0hjf0x)6q2@K zEs=zVo}B1nFeuQ1)OTz6?R*=LN#y~w7R}8o2q!Hf9v*!-Ju_#CGInx5Jl~)X|L}ranWSlkQsH~!Hi`sY$A-=OT2=S>A#1FDb$NVgYdU=>Q3Q9 zTEt4&fC)mrk^q-GPp{WQRYL8&zI-+1Yj{~|Xg?+8|g&+_^%Vq7 zGuRihZ~}N%qm*gru_DA$A0KF|$pSF|C1}Cv5`&ILooI#Ts+}N$;VHi~AvoyW_cX^{jexgsz6SUI6Mjq4>~258gl& z8M-z~D%J-I(U~_9`XDjZOyB+{wUvbKuIqcic>g=|cdknpVD}}RrHyyPG2-70*^1t$ z2%^!QkS6IvXuj<4Yi0u1916bAmXer{fd^RY1ne`$;I&S>5sh!B0@x3$B_2qcyjwLdphoN~>7#oF-_)u)*3TyRvBS z^92e|3Bv6-tLlLbQD);FCiHiDn5$PLi?BP|K>F)vlt7yT zg`J&V*9%IO=LnjXrOX=>py~cW$Wn?LarCo*mIO6jl#etSo7{0iCx!uK%$1SM#PLw^ z>GK{)9=3_O;Ck8MdbFvgi>tdHA!Vd>zJvLIUNuL%m-4@wZTHlmMVtwhId@%?0eh;4 z2T8o;^mg0z!b6#DlCXL%zJR=+oLt*8#6jU8(h;2{A%K@oybEXuzks-3FG{`YFjaNX zgT9&Cx)0N6zXF;$Pm1U7gA(p+H%}E_KET(c)zuHa8O_uM#Nm3^T2L2O zw>1Zr7>Y&z7P?a);x=z!naM?=!ruWcH+u9-^n=pS0$Mt3QqrU~*Bf=5R9H&f$X?lr zAEj0<4_S?U;lyE*!dp=o)htGAqw#@y_Jg@pFM^*!Ac4 z5xZpb;NA8H+h}L`&c4x6Y58d&~B*_<2= zwOqHw*@pH+6O3x&C+z}(g}OnKAIRhk6?{V5*(2PVwqWMnZKEcf#_dFMg#%=OOmk;y zZ^2`VKV~yxa?|08m804CA#pBqvOR8ALApdD{8OW68&rFrV;4zH71Lsi-Sk35sM@t$ zAoHbsEMVU-|pcXD4@XDqrq|z1VEo7cGa{U##v-M)+JgZ6kzQNC&9rgS6+od+Z zkt6$-kBc(v0JJ)5S7R>!TN1Ezz7%Gs#@{+}4!mC-vkU6Um8F{D?Q`2SSE_5l7NJKy zq*B{hZGLW2s?6I^e;TAi*&^lBv}QQ{s*y_wxpz@ga|}J+4F%-*k2qK&hX&#i5MAEx z?x2w6jsTFZ>njq&Lsd0o-m=S*F1c2QM(s-7qLHPUIML8fL{cK56-tv-r&^J4Uu;ro z*Mo1i+{!R3Anq?Va;bM9nVTqUs9GV93=vL{wcl0l6Uy{%1N&O6+H`RtRy|5 zA{%KVBa|^Jw!hf>u8g9hQALFwq2Ss~zYdfmEa(kvV5W zm!3n~{?5f;EXP`T_$#G=TEabm1|>yn5+u&r{(%12MffL#T9DgVh4ne`u!y&}&z_;c zYC5Ovv_SP#=bA2&I>_7A9!au{RHlQ_inmt*fnGhCiTG*fi-;14!GGKAWlqYrZxuv# zt&C~_a=+R=px)ZmPZ)bL#JyFEIbe}9T;D{oW%-Hy=hNXj>Isvy_3)xXyl|RwCJ^caryy9&EUfVpG(by(z)$=C139=y$Aqm{ zyToFHg_2Uj*W<7)sqBzN!?juFm~urkAq(*#M%iGMrqKbKm2t;>nQd6bD} zcDkRcsy|sSwt1{7K5TMZVuO@pS*5VIsNrcTtDJyny2hXg-C128yj!@)$mLa#B^gV7 zM$P1~s@L^dK@u~%m8!mvk`BcP&oR`Nj>4((*t)&D9wu*QtXEV|i* z7AiIh&-U8;JQe@8g6@4xtdImzNs$)@nVLbw9Ye}Gw-tO!<7|=1(IK99_qH0=&=??? zthl{gqCCgsB6AQe1fm7qFRSe0BN~=d+cNvgX+0zwgp%!O_b_nk2)lgZ>V0xqs5m%~ zZF2htQI(aaE*H=}pea4Nt^DgRvNb8`tFvhjY#&b3TDHEgx-k6UafR}sF$&Dl-Qn}x ze{nf@=p2jT6aXg|hGsDnC7z{3AO*|e?;`)-A?l_QK{$rfbkiS+0mVNuGwn(SXv`;D zFfkG^d0<)a-)>3)HO@#ucDbR}!r@KtjUqoNCK1Tz*ysFky=;fICO*A-xOuYs?0yj%;S~A9;r8z1?rm>z@U^`YX+QkLZS8AsICyWDJg#6wvUsJ=_+S$Xba~BQJK5RmTg%b^^t_t@gn2f>T#|ne1__sI znv#U2pmZ~ zsa%sQqRo6{0IPP)t65^jE7)s!mAxUy&VOE{m6ke=0SEtn`7!gpI9MMaJYd7;&;DbW zm`=c<=EXy9P`UW@bkZY!arrU{fhkhGH!+P zE2Art3@WKy;)N+8V5jTKB;-Z59zB^S?Au*@iH9SPHd4@)3XE)m8vv?KuRlQg#<$-H zw!zi;gccxa$c(W>*F86ImJQRbDP@fuauc`L2jkud2fgRutsKGp7!hPepl||B%b>hw zO+#+5`257G(gS%jwAJ9XNd3u@jB6p;E@iWPmhAx-Mh1M+r!8R{-d4(%%6HSN4HR3q zrR@GX?hdZgzdWCGrH+}ueACkvJVbS|yzRbU9!^efZia2Qyqvi?zteEMU*eKpqh6Oo z`f*2Jr{nmdqq|lnvu!Ycoj6~Lkmc8~r#~NN?ZYzLSG24W=DfkHEF@$sv;5KZM@$<@ z@HW2y(;%$GS2(;JGqsA=Rh)F)%TZUL{zVNkZB3f=(>$A0PkqAdEN3&;Z*eec)(sxZ z*KXe*5?ELZ52G-Jz>!{Ly!Q_e;(X;^EvGzu(-k9C^BZ7R$#rwJ)#LwdM3zpoomc#K zyWD%u8xOAoD9jVl10emf_J!aH^V_l7oc}V`K(*O2fdz*Ok0m#`1nz1tK}xGH*0my9 z5I2{W&HYY}Cq+{Iq%FV7bos9h@kmcpS;bZaNdm(xtxwH$Y;s}FnxAW_7QfCa?OOHw zRJ+oQ3SKhbb+`BTR2eHpfqSc>)?IL!vAKvq1fY-=k}Fx={b;0a8ohl!APh5{!8LVr z_&bcmPghIjf;}YLI!|N+nUk9+B#gic{${J)vd1dR%Xj(t)=Fal9%LW$((c%+o zUBsfA#d=fZkRW zZ%cA8(&PFYI(9`kL)Rc01xtd6s$N?E4+L^n(dwZw6Qq+DPoBP-j`;!Rfneg-?YH%b zX^*>5?USkG#xSd!OzK?)O=52}$=r#@revbp_y14) z;eNL`%+etgIiMl-IE#WVT{6bPHw;fJ&8;OpCXEBf(VaggG9bi%2>pOWH#)->7gwJ^rkk8X-s5{25@Ok8g}%tiq8gy##>5F9cveo0Cwq%VEhMbOy+e!0-!)|Kro zynv!U8>^M*zl`kk_rZA3*siGkKllUb{}q2gCQ09Ntt+`##*;k=`Cf^$k!;h5mF`&v zD2`y6DE!8{2#E>Ki+YaQ4Qx#eKQ=QiUXk*zHN`&8m}lGHy8;cL8rgsyR@z`TY_ewE@sJ=`vOi%wA%ohgu4w&@UBb*I zWoZ|yB6*c{qiF?Xi&(Zb!lrIH-0g18$3ghl5IL24zvetGyI1%7E@k?QfqC93mQE10 zIHkB-%i9%fRGz;0ZD)l7!Ztb&PWqg*;|Mjb&LKqE$Ix)6p<{lcVa}}6)GL~p0+BD) zdo+uIu%OO!e7~Pk^Ny(J1Mg%_no3|6MQqI_ODNEaDU*gelywn zUS0_5YM!Sr3FUIvO6p|p9~_}XAXTWS&9YCLksZRP#Uu4K9O-QgOzA5Vn=83^8!FT~ zhR!s;L?uoCyf4(lX}pAxsu5FAG7;ED`^-4^1h2AjX+A?O6*HOQuzwjNFP&BLwg zaWHO^kL5h`0G@L5u#>ZmDLRq#M>M0C(hLt|G1XxBnKKAJG7?X78a6n?PrnBShfGq2 zEclYVsgmdGIdUQD=(=^mjRvp4T!GFsg|x<|kh&>Xy|xjgmGX_4$-T5sjfb(73fu`` zs4;?Ye;A=5_0GI$_USbe=7>S-fCUk3t^{u}6~$HN{vXBhKlY>Yf60!iv;Xe6`(H2w zH%{67LH^b7LB26^R+RaF+YgIq-c+WakNXVslr9f>@zf(=;;-QJwo0J6YdZD zWh^bua-zzmXxh2cEZr-TDLbn_uMwcRx@U(WlZJxccJ z4KMJSN#uAD7NLt>{lV@z!k>(M=9e=Z=qS>bgqD+roU?)!<5 z0|gksLBJ@0SQCa>izRK!a4CfFoGY?y`JD@eieBYe(;MvrD|Bcj9G~I3YML6#;FPh= zEdlsxWAe(sBk<&0XEwC7aK~WDd&ms=b^Tcs5M5oX&py2wvs^mQUxZ^}ks`5i@S;rS zM7NUVjwK2lQv~k6K3^%PiU}4HHtPis;8%Xh-a3C~AG}t!KGe z3t;#1t$zpdUbx3ZuD7ifBtN>jJ=EGFpb_Q}>F8PVm6nt*dn&TF!l5W7fyLSDOxTy3 zLlu|$39yo|PS`^G(sRaMUALcKB$zUGc^Nn5`eIkV&RwBmMST^Q0#EJ#u**B{9&tTA1Np4Bf+V`o*GJdwvZ;>fsQ)F1(p79+ zWTbP&un8IH)9~?z60tcb8@cE+Qncc7*n~78i7B^?jx)bfnxca!w*0iAglqpa{4Ct) zeGM!T0~_C@&%|exi(Dd2nu@p%Tb0Idi-q*GiBf{Y#IHoI_7iWq2hz>!9R)(p(7~@$ zn^uNyoFS!%idg0p7yS38T6Q)!V)MCg1O75U@<+G?qagZvYk*$l_nZ3ivl|x@)Cm`# z{~D-;?b6NZzKWmZSxZrTir#79j4L2_Yb_yeLL_M1q2YtFgvzp(n^R8~KxD5bF8Sg) ziSAG;qB;k{kXY)oCaBTkui-&0X|z9=Y2no|OgBHQD&^)C4gr<@6d8C5VnMj%!=s8j zz$)uXt0Sa4CZq;a!S{gKtx#*l4r+4svsWK zR_-G5&Ni2wJ}qp@YEZEfH6$P9n!gH@BPxTUjxQk*6Hgj_z4{L71Vk|UjhcKgz>a@U zP$#q(?Y-TdrR{cEsWWI8dlV`_JLRdL$4@I1{=11`wWeVz_yH@1#Xvyvn*>U}3p9)* z`(D#3&q%?jS<2$WVUU_3TPdE(wZI5;G9-!83E~e{;UimOv%-Ky83?< z3R7&(H=BUCMQj;bl&CHz_Dh~WCK9jWF%K$ewGg?r0O)J_r2tG88QUB;6@>g6HO9)5 z{hfpR zMbFbxnn{_=?>k0pYcP&3{^#z-_eG&STuc_~@OXY9UqGC3Y6r z!j@VhBdKV%{&qdsdsh`>)uTDd!U_i^!KW6J$%z&ln>&Cgs+a)TYWzx^zS=Usho-)?nj)(366 ztUshnHM|fFz@%7M}N$dm2y39 zwj)}+ZTSM=^pr+`IwaOlOv2kz=tYXvruXW@32ZfAS!J#oZ9Ikbv|D{-5vVJAHg=o~ zx%301%+bgl4>Xx=525w-$so1YDs7>JTj3MUmt0!qEAupeL)&5%LP}I1+Ng(^X(w0A zDXB{gw-ggd5ikU%t|Ufq_!4*lOgN{gRQ?TuWZ!7&<7_|mJbalXtrk$ zb|OdI$6nfnM!{92tmQk+h)Wt_um^Y7MjqGBK92Vf1DiM1xR9XsFQ{-6&yI}|tweag zg$UuE-`#@4(Ld(VARGwN;>&Pq*LHGbcEhLCAiCIxcT*}}?tdRK=U=AEqILbJjFS6N|DyR2q(@(OQ%WI=F0cj^jKvC=LC!aG9SXe;e|-gUv(j1;UWz|=K#iw z;LC<6l44B)%H1lqd`7yYJ(ZL&7TrT9)0w`M@HLeRsvfTV0#UJAc;fp&pp*&<_IBgI zWJ#T*h7IPK)gDL}zFDNfSJlZe>~y5?g%NBByhuF;96B_f~q8cUB0(xsaMi>yc;)1B=x2nugjpvFHTevEj4cbrkVgDYep!1 z-V2c!ls|*OU==SR%wq?ywFp65|MstOnoQuAoO=^Bskv+XA|<^0$a_L~-pjFe8pe8L zz`w+LYuouQtHjgr16`jBq#kisJl$}=UH}AZmA%aQ3Gfv$Q?VB6+=3@}So#9IBJ!JH zFIPDeDwJrT3&a#zs+f6M%xOTD=%yKLQtCsAchgWMC`OhwOJ>itk;aa+9(B~Xvsx~w z#Cw6C5JCzprrHh3&eRTS4>a?T?DdJ|L} z^sL{tA>6_FLhgv^ErG8B^GPle4l_4tuJ{R3;D4o zJ$sy{wz^$PfwxLFr{t!&`g`?RedQZgW{ncI@;Zml?kF{Q<~;!E=aR&?$DP7}inbhB z));`6`vwNSx9Dth3{BY+TlDXU^|TjgF?iMLk?4b{K6hW`-bNx~kvPe#qbkKp9PiHpk2vcCS{i6Wz2)*n--)()_+iYB=@q*{oO?Y3vlVS0$+py`#iA)i{4FH!!&MZd z{ifvV4SA?OZ$LLT^%Gjfc*(hG`)~gEXkHh-Qiryz^LwQ*uWOPf1Kg6rousg2ce4Qq0~ zH~qB&ONGcG`)7x0@si1je=>&rLMHI|aDcI9dZ?Q7V!cKgbuJ@op~Az|gJOc+^~JA8 z-bmqUo2T6WwAbKr{oF8Iniakn8q=62sR@@3p79j?U<_}6dq^6c|2o*OgGr+HD|b7* z$;x9G8YK9Ho$N(rjV!HBIYWpM+-(FI<@j6NwloC3&vCHbwE+g>T1I3nM2T;v{%0{W zU0@R{zKO=!_1_J{O00CwloY9VdG9PzfC55uOY7SZ$4?o84El(M;Bl%^bTQ)b*v38B z2}cExzeVnI-y4yQnYFOhJZ*L3Ih!76=jc_)@>^Xjm}idRK@|#&Di^4F>a8u3PsNUN z)37#c4`;hfdl-AIM?n|HCrc8mR#)1~wn@X@{N%`A+iLn)h0Y?uoE~VCttzH>TZ)G` zyy0W1W+=*mDN*vfz&d}?^jS(w|MJ9rdN!W}C-K&s1HUkx1J9f;_!TRT`=5)zO?~hm zo(H!Ye^r|XMu~Iaz{a6zgyq0C5NfKG0gs93y^KLSbUJB61H<4RIh}E)hhc{q;=uNx z3HB!$=D^1KStk#(VS{PG^c!ODJFPgq7+SfN{?8_^qWIjwW>ya4<4S%I=u#{&`o(z7j0W5TSB|*Af(rN~%CW&<+!=Cj z=}ng<+utAqxX|*S5NY|Rb~|DT%WD@cNnF9{QALZS%eXN-INE;$DE4YUT<^P{3VFIA zb$nlPr&7Hf{~9@u%~eapb(5>)VU-asFb}V{@QYlvpEOu7l@*1D39{KpAGBn`fCJpA zQ>0Qw$56(Kx>(zAXI{n|U%0faaZ|BC&9t>P*rWBRw_PQ@tH&-!t$ZtF{Vv8Bep;4* zZH#(dZNBA96PlhhsLtL(KzgPh{0mq?AAHHo-tuR8A+olcQ0+!H0aiyt2d*-SaB0Y{ zYI-x+RO&jKqv+FHkPl8pWkqnRES~g_gSA)@^?4+$?vuZ(JT+fii#?Jr|i6E3>- z6MpmNX;Q2--0}IywRcgpgPB1tm|Mg-Jm7+T-%?kaW-5N>6}AbEHZN9R7rEDvDvV#J zVo9usdnV#PCDT-xW8T}Cj0Dg|$i(Ej=3@F22J$C`K|C*2h5-g-mIZ))ND(4~ zhj>&WnOofmpd47*!rox#?^O$Vn4*OmGF#)VRObOR;h_d*{^Cloy=Ndjag7 z-5_?OX!!%2u)3!Dsk_bAuk2h*79MFiRf!@c<$8Dq+xfA4>&QSZ-FaN|pk6KqaPZSy z)JJ(nlX^}`s#K}Q9O=KwZ&Kv6z;4(JMU^qr4LR8ZM>(CQ1v2u+ZKGFy10XR0PnsAa@(D+7!i^BdUhi6`g6E6$+g)lT#}?d_8DtYbsV>GcOb*C{vuBc@%9;4ZxrG zLQg^$kcX$4dk^K@sF9K&T5gW|`L8)n!{bOiP-?5nV%BER>T zK{(ooZTH9QV!19GN)hQ4r~I0<0|GKYU2;uzfyPRUzsI0~L{nOb-01w$Ck|R#6iNIb zwN1;SDder^m0xa63s2v1#oe;_cm-RVG^g1R%=Nv!jxfD(j^NC>YNbC1zG0`+cgeB*1>)+Kz%t)6Y!;tH31&mpp` z(6&P{n+_9&mpimP!ibruS6r6!?~lAcCB)2y_|fv0psQ1DYhTNE6XrzTBws z6XscC!&aO$gEthCm0H#>ed41NIGh@+RJd8$RMc)Tta3Zn(o6;bk>C-wi`;qB@_v6? zPdT8Hu9b^3fG+0{bj;L(Z#&%1$8{C4af_Q0o1E`{p(u&b;MKSIj8^ZL++pV=hGuze zdsGJKFoFyrjp4!;lm2vm+8d_`f*yR~Mj)z!gF!dvw81@Z`W3NzIj{au-OvI%@h82# zz-F(4d}Ka(i=s}kz#bFj%r)z^`L-i!-axz8RbSSeX>~C(lKTdxHv6e-(Ns~Ei zH$p162OP4zy$O;vB~^lDtYfxPWpSdmQo%?)^8bcyNj;dDiFCVrAfi0*<#ZshQqA$* zAAm!JrO?hI|B1tZIMTpUg@q|?x@$9jh*H3_QOoVMJarj2gJkB9kDPVTseQNLr`Z`Z z^VF9G)8evkf)jAWEC(RO4wWx5QU>nu9c2(Xlp93zMCU$U*AV$ju&6CDlXnr`s*$T; z`3Lrlc+Q}~u}!3d$7~7*INkK}mrg12G=+*wK=NKdVvG;nGOl5Fc_$emWMeoCQKh4f z2zjk^eN2xXeIHJLt{jAj>?zX14+^Z@{xes{R0}|6@e5d z20Shih28jt(g>*OXDN_#h<>qKz&){XeH-4?xyjIKuVIh){+{lQTMZ(;qW5Qz);lz)_*xIV$;iQhi$~hHg#>j__ zueHP}M~+9Etl_>Z!4i=kCyTv>B0$e2iWd`(6C#Ts($OaZ$1H`&iKgISlaasH%nKHq zJQk6_=|2&IW9EtrJa#8A`N|f1PE5+iE+L7tL zRdJGAG;K|O<(iFsdNI_T{uU4Y)7WFZUy%}BnztqpS%$qxHyn+u`Ay1VIx}yOU0NQT zQJgL(=C-#3ag&~UiDYx~K-x$SKE5i=++ENXU0kSZQf*LfTQkhheO2^qR>JFW{O_40 z(@WJ9{X*3*>3lG6E*i)E>Lh+-XRw>_4q&<}-En^h6(#}Ugq@x*WktU7vB#MPM#U`X zU)GiqUMR>O#F~NlxsE17SGs7~gb8L?;hZYAw2joIwmmmmM`Q_&gc&Y+Fx5($l-V=k zX-44^r00BZU(}S&%1R|kXQnRV>#55x4C)2I(?)UGQanqH>II|`4_iO0PQ|Pd#qTt# zxu<)~d}(_Wa}J8(vj7#${GkpG;L}MWR~?vw$q*}6!c*-9dnLp06ysW>R^lLn66!Mm zys07qd9#JXIZQ=Bb0*)+Ocv(W#B#DryCE>*J-OwwW+v)5C634KNPna znVy1_vx<`>7~SnWW)LBo>LONTcVZw`Y!wGD&PP(m$Mla^duMWrREti19^x9ZKkKwL zWhAw;voXAao47=9@KkDMk*Lw#Nq9HU+%`iz`oQ?Mmi*+cg2hd*gX?-X?t*d$m+sJZ zschtosLNx&Pc70BIloVcuY0{THqGC(v$clk*lQy^Vugwz+oE|ce^wDn?N@n`s z+&PE-I%$Gfj-ob*ro!T(4U2F=5Cwp`nEWnll)7c3K|NTvm!g43lQm$+uF&PO4^fJk zq7ekvuorkm&G{0n3+v5lW&?>>v9rxs?o{-`7HNFF>-O~H7%ArAh9w$2w^VdpXfP~L zYZ2lXKc|xai0iN!XS8Zo?{~hGriaG0s;j5Rk7HsW4vdV`F}!CREgm~0dVkG)E3Vx|JW3EnHY40bnlI2@Je$N)3t*e&xS$+f~YLOb5H<{W4 zC~f=io{?WR%uF_OCd|x@6yAN6X&x(LC%O-b$Nq&SP}*OptZ2Qva)V4G8CT<}L zQ)Fp#VNt%oj&rX$aP9aCvBgmoN&-Op;Y*kK%Lh42rnxD>8j%IGz5ew^-r*WPAthVU zVdCsuKYbHlu<$=9eHmtG!%&M(QxV}w%|2H=Y?VF9J1BLEiD_`Y^;S61JI6L3j89D%-`Ctb!g{(1#=Gve+hHrnti zF11}7ktfvcr^zF&SuTcAMW z2N13U-xk}XOydJtM9SJ`r~@_pORc`t)^4Hi`r_qAj-%lZ%=zX+5ifp;-A~<2y~XUE z$L?8{o&{6g&p4Jh@>}w7;v3^0jSi~a#IqwQPLc`q=nB;j;3gQtS+weqUHDOWa**6? z*G5ehL=J-EiBzi+qpYrxBjL%&;*zzS7QOR79oJ=IQIR!0ImS;vb>m4P0 znIu(ExDpwoKT3qfv0FUjswsoLnE2|wC4BX)S<4-Z2Wx4v{R%%$hZy8WAZ7_zPfaSc z(?w-4Ii$Un(yF<0hy=dY{+2v;`UJ!CCfKd0CGgH9FBRmp7{7sjj_zfS^|=SotpUw# zErA}Hm#_2DPOcCmn9{dX=s@!pOqZQ#aO7CpKc+}~IGPEF}P6&`M4G-!rAR-7X&;n20EM6^0%u({Kre=;eEeaY{C zl7MvD+xSYYdMZC~H_1KTjP5-A?d08Ni+3Ql)=Mza03c7FwPPA-GXb(s(%P5Dfmzd}TUFF_vlRrg9Yb@M^F0IDK zh{`qZbKQ1LRc`DpOX)NIHDpY@#Pi72LJ!I$#3C)pD8NWQB8`8wp;#0@LQP%xx!W^^ z6Mb=Wp-(xm+^jo9JzD6|I7<~+X_@_L=T0?Btcz+V1p#1rW@dZ*(j6Jtga-E4ffD<6 z=BJ%4wCQ{dN+(}Jp!jRlcHnwtr*Riu4AOi>E-JkVmP)0aP zTUi=tBCelJpRi6h6(LkBhAsD-3Kp&Glebj6|E!)~rS{(cGkkI*kbRM=9e#V2GMiW0 zW9Ir>I&gkGXiVbI{P7{f-bYlkHceF12F3|lf#{wNqVnHwFvTrLk{h<$&v6r)XeXotuSs4}sX?RU@dp{RNi{c3 zCD~xwL}ogMBu_i*o;1I=!iSS%fZs9+t&JbM`~C z!Ias6?|)L2Wz?Wo=f0%Uy4L+JQhgTvXl$pG$CAyL16-1AR8WMeHiA#Xc578EhJ&6^ z39wRcMKG30R?-%bPgSFIk#*1@#?+}0oVYOLuj*d6QBRW_taN~Bad zp|73O<%76=JXf&V{wOa>yV}7VO80f;glK5f6+3HR7HhZj#eySGjTjR z5M0n?jkE@?_CoX_rWQU)THaM6n} z{LL8jM39WL7NPb%u@NQb8F#&SbOpNgPIJf2tXx#3G9d4Xc$>Wzp8Awdd8LZEC)BM~ z;H(1v5KJTQ74p|_O2ifeUjzF=L=oN@y`?ZX4A6`DF2gpWqBJufN%^LXFHV`LpGyd` z`jOdzlx`=VjJ*=C^J=?0^5^Lg1BYLR&fqt1jdRv)e9gP{XaT}Gq_(2Y$e5lwrTJ+W zg}DN#v4If4F#>zs&p#d^eB7J{c1j?Uovy%nFO%3&&ig$1pNCjY8#Y==NY=Hrt+m~O zKPZPWK^kr@vIuQKTG`IzXVfSPh0TekXDgsnG6#3!O@+jjLhJ^&G?T0(RcN00k}8|@ zDtu>Tm1_L`qI*BddU9($Wdl>g7Ip)PRTu_-%i0+8fI6Nq!lut!D*BvMEC6n;9U$XS zNW@85&gPEaYgkn~2F&Mht;KF(>5l8xW0_ri374x6)jl8}61Kwn&|8)*dWQ0~fy@PE z5~MH`n{I@TZG-~yI@aX`rYY4^)8a#A;Ykif^;t#+MH?GJ{Tyf>c>3IWv^}nO9<56 z>rw*{JsJ^K;P;DXmXPsbGiPY9Iimn5_AveROtGuX<~;rHp!6kqct!rM5N|PV=WN~Q zYZ;*XidH*tR=>w@%PGT}r3Z#)oekM7vP^Gzg#8}RaHGxQIpgRe#EyAJv0>}HmxxS6 zOF>aNFu?8e)lU2zP&-daTxSM~2{-#W!IExzap|#oq(v9a1xt7UqN?(- z?0pAxylzGDZPyopb&bOFzTZ@UCoW0v$-lD0Ma)yxqDcSWq|?eh)OG{`#r%$_Zn3zL z(iZYkc4@Ut#%j+^n>5P(;3G=nCD+-FQ-!!oo&gh%NJ@fWmn_N)p+K?YkZYXHYs_gM zrM34uKv;_ndvc>VT8sM6mGBIfSSE@oqn2@?$oIF<+*8`Q$?;Lpe|JYu^rifnzW?^0 zo{o-gce_46E)J&o-_B=!=`a3){X9R8@2+S1im*9)e4wWa>}c5PDmy96IDpYhsBqoA zb?~jXj;Y!(^x|#`yso5P)-PR4A4hBH=!+RD!>)=*3ZL_)%9wBcPo0Crhx76 zy-vIQen|Kksu7$4QLz+w_)zKV@0v(sed^2T*#zd$zM%><>)L{?sr3Yu{l0C?b=Z>o z?s*$M*JDBDd_OsL<@f!1T%NV_>ESr`<@}n|6O$ERLhYnjkATN`c&+sBzchxv-3+8( z&zOuzR$Qel63u`el&=-3@@LmD1SwmVal&yYOG*+LjkGZ`KI_ll<*5}N*W8}%xQLlzC+!u z5ZnNm8`EN{I&`Tzlaw=MUB@Ynwr0DGB$?B;NOV{W2+&;i$EqIqo5WgSOuV7?nz9 z3osIvvBNNxre~h&2+e&;KA1vN0_>a=+Tq*ngqjZD`H}9K3drW$#PAbb;|8z~?)MWRowW?UI_FDQ#F;^i~ z-CzmJOw5<5Upq`OZ32(I=6+Rx6CGT*hHtZ|uAM^fKY)#lR z>kOsRISeRF9z&xnehWga&(PM95uu}hdGd%sdRZe*M%-nRT6LA6bnRPh`QLbcL`*?arYpgH0i z1_@mwwv_Va3_V-};1z$UComh1S=v+<22jB!U~YC!rPTR%*uh|Sl8xlF5fnwSqxJWv2{ zMJFloS6N*oYCg2CcQ3Q83t6K;*uQ(JUCtUuBfAbKi^{vBR`AhMbDpjd8IjZ^$!Ux4 zkpc!UjREk2M!P;?fHYTG4N|Z=KV@1m3@F#FPR;BU6?G6VV@Oqqh1sBYU3&D&co+2N z)JcThe3lEzNK}S&Eb$Sb&n0Id<;WL?#3^CsvLtp1yxnBagv!!VoncHx8WawrxH^za zGvQ>|A*)mwzeUh(HkW5W*~}TI#^lPC>eI>S zSf;9|fTKp+4dN~~3hLZ4xK0rQoLNP!P&Z?SKyO|#EzW!kzI1ROd~_Xpn9jmvB|>+A zsL;SFturEIbqQiSo={?l67wwh5&)^`DMXc(*XJW9uaB&l|0OLRT+=Tr9{hV#kAvEO zkDf1q)a2&Lh1C3fJYY)f!}9Sd#W6rqzcxcs*9pu9SqbNn0ix#losY2yTFK3A0MCL{ z^9ovx%3e1fh9Nh-fK+$?c;(D(_-IpsuMGS5DIdLGe0g^HfE-V z8jj@aP$vybNr~TeV z+ao$|DI~Y6LYN`Pb+=*AK@Pt1mw*`UR08K5TV|e2lE<9mOAykW(a`B55)_M_waAp3rH~-U3RZ zC=hM<%pX5}$}M1F+%`>-prwvyuQMpi00z}H4?9gDC7RWUhoidM*bN*lCf&*QbdmW5 zs{VvJ5|bqxDRPn}5BmJR@6uG@>B=$nhndPT631@F$%kEK>h9b}LKc>v-uKqNcsepe zEQYCz@igVXy>P}=w(fWUea?737HBRu8hOlj0ZOv?&|LFoXzf-;>hd`4nx=A=>J(A* z9LZP_>SRN{Z~&fKw)!>;`Q^205-O0VCK8w0KjPu3aT{Pz{<$m2C6I~WAgN?VD(Wx6 z@dc&Aq~&1a<*A#zl!#+kJTq@em?>ISB@2PKS8CjEi+gJU=f6mAHLs5ijhH&;Lo`Z( zURd_M?BOPsN{*_@Jn_(u>og8d+v`#fLE65ZCd1rbml4})vuM9h=n^i|-NWN><8Y(Z zbLd}bhVY$R=|MeRcz1D~WW3X&)r&oD;lo&W#1Xu+X3W+_-;+J2!+&NS<-9f+w~MH= z>1D#@V>Vw%<46ic4H5%2kr8Io*DQIJab>(^bF07wr)FYZeQpRun&yUQ7OB(p@xoFW z%#){V>a`VS;{s@`izb;zAq4^Oly`Z1muHx_yxj=p+HveG>lpamSfoqy^SOGeO@@_R zoyN(<_sCGFT_URR9xlt$ijL8j%s-9ay&u=U+QjJWk7`vN%txhFl2-#=9?`jfu`tgR z>5Fvcht<>uBoN`dG>UX^}twmBK8&J=Aj9T+f4tF7|D-!4d!Q9(w zREQq5QB>m+ja7yt$&j~4jMOfT-o}H)Jp(Z=4|T?Cr4+ay;5?ec5U;aM`wUqlJVQ`h zoYOPXoFCBiaWA86L^}vliOfC-242LgvH=pmN^~1w3=_3!4QF`6ekb7u#m>;tYIazl zQnepWz;E3yz$sodV@^E#2XH`LJ$$~~7`scYdO8CQ@#t?U#>*L{#*k!x#4;tv`6<_M zMM7DhQ>K3B2Q1z!z{|MVTv%T~pSulAr|RL3wAySg%GPak8>4iw$0 zP($ZY3W(apWt1g52PBW|xKl)`v;(~A7az4KzPcID*@DekHhN`WLxZpx=%L#0l05bk zw(f~gUFqI3?30(}{h_?Mb^8H35isR&R;p`2*zPod&*#@%hoRtWv`}4Xn;DNUm<#Tej=6l36a{Bv-~mC9?2Dv;%Rc=}Wti14s))siIe#3#<9hiEQE)>c^#=1=OwUr7%-NjskOcm4G z9}8yn&@G#Jhmz5aDJ!P&v{X0SW+`X~b1G#TOM#922No5}gCV7o@yY|A%u(qAT_7Nv z*uRnIxj3+HLAU(F1|53fsTT1-83HSqfq_g3Bit#oLc3F{M64K+T-S^QxcqQ2>1WR~ z5H-^ij}8nA4*8ByB|zU)GY!05zcYA<$t|@@?L}0RgaK9XjWu0UyiBlkS*(-85~|F} z)};{{^Av0VdJ0trpE89O=Ue-gByRUM=W;wT;GQnH))AkskQp1~L}p1mCnG<; zwKy0UV~pj^cl*TDw?2CoEP*dr%U7(ffRb&?`?!c2cMMph5pH;gsWPMF6&Hw5+Y86P z$CkbYL^Z5@$;dt!$guv}2O8pso4uh$0cf*ng{lHNySzS`9?vm%$;#O^%s*{%oX{%9V-yBm^BAtBn;O)@;5 zg4!ak{~*pzF;=f(oN%rdPVTH@OpAsq0+QvtGH4PcVcCSxa9=gDlljQiu#euW_1_If zF6&DHyM@9cdZhT&=aYXDR`*m0My|hLX zU~kW`H1S2*s22s2dmx@29IzXhy-U1ViOkUi>r4$re`AiQ%VV$QQ$5bqeXNVuDaUkE z@`Tas1~`ISbL54E+jUH0L|q@wwiZ|}au?M;Z%O@B5`WXKziJ9tAnU81L4Egy;KEX4 zz26v_cqoUphq7)`O6xDEh^B4;H}(=?^MEaBEw#>!eyhuobmF4Tw@1|wN6w<7*zfsr z-AIegy$`9^Jg`p*7MDj&X!ed29m=H5+*=xd3igNv2dym3TW(NbTVPCLy~EZI8nI6J7Wr0Wpc=ze-x4oI=MjwE^`ucIW#;(73v2H zW{_WQp?R3}j-lygZu7*@0}ecU7)y*ZcYZ%K-A(2P9^!OC%sLbE2<&6bA-%w{gg=^M z1VZ7$DG1SkX^_=F9cD^F{0{3+cNrP#7X<77`9~`cbcRhVQIC*XH7w2(djFn<>?c zHur*=ORey4ydIFEiYa<&IRTYiqaB9&LKIGlQs^E{w>a9z=1q~mcE2jGxY_zH7=MMR zKK0IJAz()U=t9BHlXQJAl{u5!+)&d%^-$5+`oHS0fT9TS59ma^Ep^J?oDg}lc4xF!M)O$pwagwxu;gj*r)V1xD4Y{RxQbHd z=?Pi8mGSG>8f@4U@I(>q&t|(lZ1_2JWA2r@Q%e;ayk*2nD-88O-*-7Gt_59?)gd;} zYE)7h;BwLUJ|zwc(QROR6$`l(T-I)jd6yH+#I0119q4!~E(Y(>e|%0?;=~Ba&Z@lO z4z2yC@`T$q$r}PbEPwPzZZs0YI?6)ir0MhOe)>we3zseN1!*vJ!eT~}VmfD`pcE&dXCFiFV7P0^qe_k9O2PmU1svgl}LdGmpO zjvaLz!Ugy0d0BWgGXjZ5{*E|x9Kf`(fD@R0!RE)amNfFRI5abJd*pdp zzeuIWNRinuQknCMRPqK&d;ABf{7;D|C#N2h0&iO;(t8`if?iOaix12Kk_TaQG@1fr zaitdrL}}lOtg=MF?GPj{!P;V-K>x#z#%~h-dT3$Osk+BNymQqmYXpyWGU7K z%&f6WSlS?xgi@OPV&E#l-lCpU%SEX=GEC{Bc79X z!aK_7E{{R0*%-rr*n$(^Fw2Xv7W{-H$17?s7KYa(Qy8g2!_dR+RRgu3Ge(OP@p|WP zec;cZQ_F4+j+v;}2jlLCBsJm{y$A-Swi*(&*(tv^iUPvFYMCY;sZ;Ed@%~;*SK+9Q zh~3r-Sm8-5ic~EgfeqFv z>IST|3j3^+EJ-;!7d?VH{(;V=vHds1^&8OJ2ADB_Siis>MMzJ9lzT=di)!FC>WHyF z_(D@>LL#f1`j7l2<#bEwG!m`TPVI(&mbf7N`W47(nU`Yvt^+3rs0s zp9KyVtQLjMyB8Y)O?>cxpD`gF&VtOlgIygBsf?jibJ{>FQ`ixd>6Lf(Qd@~z;mnoC z<|1<*=4PhI*bibB+8hp;7e=2*p0H82XL4aI2;s#{#8%^!VEypS;0D2aH5&MtG6q&x z;dTO*r}0+XDsvo&7__ri7T^uECLFZ;u}f|@?P^$1#5w5 zM8tvr@nLWsU!74PTc;rci?yDB{YNL^*HJXPPvYZU;`8iv|1TE$A(=vg>_*)vCZ#4# zSD;uPi2VpyKXCR3gr}uH-WXKJv;=2=wMw5<&ZE((smD-Rg2-5g`tKd1wW{~d-3BY1 zVb5!+aQdxZ9zv$#Yapt9bql%*aUJbEQTC!VN|d(O&xBntNG>=BoF)iawkoB5Z9ELY zHZGTjk73^fY2XkMwlz#&QWTi|ub&BXAG(F%$I(4_3i$YyNwvRfRec&BP9f0@Xt0%d z>OiymyD@`_;C=e1DnA_oM?O?b@6{(VYI>1EwUe1^^Ep_YKKUYYlDxLL9wdO2kedvq zT1dLcdmN>j!71cOeyJv@Nf!VqGh9oJi-P6K0$EYmAX^q7Y(M8@Giq9#6l7ea{!iLk zBM`66h3r9wDjf!`q2ZMQ3WR2$@VU{-zE=)Pfrnv?lYZWbaBi@u>&T~^VJXC8?guIgbCmj8fU)PR=~_UOnBC%N-# zBFM_JgWm%=VHc=fMmSeOVOgNt%eW+aFW^f-jy}n8w;-_)vM4MaWs48iz7!i=F0N4f$2B4~NqP~kQ4w-B z8SE=tYPRLKhEAq!no>rSyxVc)aXPd6;#Fke9Dfaxcl0rZKY(P2DG;E5j^4<8nq&r< zN&TVRi7{aX!%G3S$A2L41c1lFmwMIDWYo9*xvu5kT=q1vKk}?6hRK56 zj#xxZV(0$B6O-amxu!-g`xF;5X=A?UMazBr~)KPc*DgNq{77-^npMXOk^ciF~j#JVM#xFOI7#%IHg zzXpYym#uf*j5lw(*vl)aUz}ZnF2f|YwZ$!8Lv}YX0kOZ(U^kjwv=gSup4uI;!VTxy z`!&Q21k}QP1L$?lzs*n;=>&2)Mlq_7nVz%~cS4Fhco zGqXWHFTyhrBPus@by=FG15uvu{lbXtf$5yPeI$+HjU+Pl#wV3szwEiyXH>`Ji+z|;U| zwur?eAOE+AB&3;ryEP~>3q_u=faMVkJX5$m>tpvx4;e2`%5ntC&xVH z=F^x!^9|oSpe8HX;^{t-@>_Kr4QXcbhBN$0Wk@KKxW1Lno$Em2tabx>Vd^0&2#Z&; z-*s#ns=PGQ4y6F|i!&TB-9Hg|sPR5`(0AYP{R3IqMmX5mT9g-tq32aIrv?>HLTVIr zDydT-DyrDCa3#l9v!)QZy{TX9{#^&CKly@@hsZQJq%&nxb|Vsb6NpMUG(iF~?IGmq zTp6KWN!xOECB41IonmtF17FzU$jD;XLnks$0K9|oKr~Qx&-uR;k&>6sMlL%@c2XU!DXy;NlUjX0CC`lj;GwDuJq=O0 zaaSbtmVM~x(&c;`F&3N4!;Q#%)EiJv&dC<+NcpK5?{DhM>J($AJrwNnGArXoGS5BKW}$7tnBqbRNVo-%VS6Am&3Ix{( zbSop!AAOInjw0PRvd{Ssf;K&urktK<52WQYMB*I6q^ZJ{)ltRImgU{VAe&0@L9Y7P z*#K=OWyX$0(sFT20SdSw{k7PFLCQ*sS>wsanviyD<3yhxwiO6jdrsP&m(He-G%1da z&^0gfP~J_2>}lxunKVN1gZBt8eLo(*&94(5Dd^%!Us3myn2J_>CmPLC@S>K<7&-4G zy=uaR>?ZT7O?f8Vi(k1UpX5=C4o4-mGd9M*+QkDFh>IR`{qA{$C3TKg#w|FX&;+Y8 z&u9Up+HyT2mcY{w`5Q62X*!0#B`Hjy%K^q*n(J_<^?UB_1{s@z@1F|G4{5HMh}|nE zMNjLG^ZBG!)Yga|U9+VT&>W*aUkZF}-;Ih|wJO2%G`{yg?lCku#3`d47H&J@soW5H zKuoJV@)ERHXLKJm*Zshq=Wm<2te5@P{Q2*=!;77QWU=_i2?DTrS=R6hrDd#J=PKtO zOCsa-FSbL-+JI z`v&I$$>-debDKCRgmADt83tPP1(4=D1Y!~>mnIT94M=jC&&mPEPYwfOIOkFBV+e$t zu|!swWoC#8tv|>?^=h zFEb}4#p0p*T{Vjl^L46ZfdUWBxs)=dhAGq~qIrSKLwdbsl&zku zA~Ovn4N(nildqB`E}6$oFdJ6GJ%;fBC50;x^q)_e1hrAFRQCwt^j`-n0>y)etRm80 z6BF`%;Uq2>=d9C$@z#=Q``1anl&}_TOmj{SZ(2s{ypA}?&5E%Wx>li)=yWvCR`4iR zuGO)k6@$NKTfBZYejvPwC+;i3iSu+ZXxzjj?=(2%d1{ZUp`)Vk$cuKdv5Yl?4?vVI z19bjquA1D7K|ux}+VNGIPm+_O6%{i@5+KML5gl`89I&cetg*hN%Uv>bE{pfo@Nxx> zRW?{?dgbex(iXJ0T6~SD)RXE=bz87X^ilEbiZfZyisb(r%g!sBVx6oPglqB^h3IdH&0B!kDUaXeOg%m^Wj>^;VIbc z7DVl}Afb7o`!08UjLF~YP*@Hf@JH&86Z_JoIc30Kxm)_i7?g36QRYM(g|Vws?EoE; zAWw$xTL_PUwFw{VU24;1voDF`^BZA2XzumI>yISVd}B<4^PuD<&fmfJQ}KU#r_l4} zT-i46HiUG#UOyF-<7&C^GQu)NP+F_12-=Kw?sB_$u*BvB{QUsCfnMp2 zO#;h8HTooSaOlqS@kHc7K} z8HOJa%blmp8U%o1c%i#vAy*>_LjKFI-u7m_uWF)3WY07P8sP70?|QJUKm+qDJEs_2 zLD{jb()7@+Vh~w#mBwNrQ!SFuQ*0!xd4kfQCqx!% zP*a8r6yiB)5dwMAwwWAcV~ZkQT;F2HTD;6IuV2To&Zx*Fom`_mz;Z{6L6gzKfBMs@ zp_+9#4%BeZd@5`mHow&%c1UB7`$V^dbycRCFbBKoJ{g@!UaB=`jGaXO`F9RXUxJz) zjFpIml#q$QW{yd|K0$@ltTk%RalA4Ch=J^e+P*nX0)6p=%GnEesKj!4ue*UPR1Dgt zj60~fhU~D+wLE0mU$eonr^bCEM^kKoUQSH&u>nHrncl5qCm&QZaUHuQF-5I4o@syr z1XbhNauTsJW_*Fpuv3 zql2}*qpXW0Rv}n^5NgCaA!+Ubm=4vh0xi*GDZCc^fc$piX}XN6xc}#z2NyzB&eaT8 zoTb4Co>fY*69t_>##xu;rRyA9uJo(3$q7l1<<-H7=&0qOKJM+08Nq4l3y$T8_Bm_d z%k!i*#s;Y}{?(n7tBx8DAc-JClRdl{7H#M&i*R-%xNK_;XYMAE@RI@b*{n0KFpt-hfGJLyT@Ld9{e4&^h%i&>*ZZ!s8F+6NC~MaObXD{W)7VI&h_AQNYA$-+ize*{dt zq0*}4fZ|!v@TfcX<$$7}A@%F|KuK2III7^*RPvt9DXP4G<+mwBzkV#Bb|~6lc%Vs0 zd()nARvvpbIEG3D9R~rOVL%N~ ztliM|2;h1s1$5|Ba~MAr^U1nlIY8ZKO20}zK=WnsJ}Rjj_=|jlx(aZc4D2#X{1B>$ zwh)1ekvxMTU}e@fz(VJuQQ%T84l477MlxQjg+{|sEXdaZA751Bq08d7NCBIhzn&82ZJyHrJ^Jxhb zE3~=R3rH?v{@7&Sj{-?yfhHLf#2JEQQpAx!e^V4pGYBMLe^3;>8`zVuUnz(r=F(b( z(MDr{hByNtD6Zk~;DJ!)+{c3RV?lc@(FhL_LI&auo7RgBYDW1pHtdBHX_5&-6!MxE zl&Ru_>9L@7n1fg=@EFgxnULt0U_-01U>zFLNGs5YD1iT^@EDMFEASr6KV!oO&oF-) zhzTT#2qDG_Ul$dU7eNCP`&aKgi17RwVY>(@$O1!+vVDQ@Z;qfy-Z>IJ01pe2+Zo_! zj^izxQC_&7LAa101kO)8JbjZwJztnLSD}*`O;S7%L==V?>5n-+0flLVAVfYNhkO?0 zIygQH1Psdj6@&9!;86K!N(nrC2^KgI3l=~b@GY`g^uOeOUn2Tl|2%$hfbSsGjCp;2 z1LfX$0e(cdBst)R$2S1}!+vjj1>orzz!!&s7myDOuxCht7Z7?A^f_{%8uB?q_$hL@ z3i5&L0vmyO23~)%PYt1Y1m1p7QGi^=JD&b^C$X&rFTgf492z;(^#P0Mt&|r##e-O+=Xa1B)u_#8{v7h@FDB&O*C?CefRD; zn+c>5q1{)a-ssZdU=iQGwiZb&<(3)eri^k-JHflQ&;f7OXN3#pv%Hy#kdSrTVB)~% z-sWa-3ZC4Z@4zFJ8ZfQKkcX|(F`+BM9r?w|2z>sGf6a=#CFd1X&(te4*|zr{#Haj~ ze3$O=F@CUDi||SkBA8Uq&jheX^o1i?lnzwIf4$(Z`(y2g5XXdv0jwd0*Cam@rF+6| z@r9g5Hou#5Lygltp<-%_eg;uh%%b{X7eSumXi1+OC&*d}huSe}_JpE``YDb~Zc zi2R~GBu<48ah$)q*awkve5FLtjv(DtCMDGcMa=iBNV%tq*P@VCs^iP~@ajD)Hnd2H zd^2uOAe}ee{_){~Zt!j{ERbN(4%-+}q!R;)BNHg2PeTN>aEx)D&x!GRnUA*}s7qo2 zq%!oKf^k$U#=D%9DLSAZ88N!G7Wzp#>$j_)JFVoY4N$r*Yxl$3$IV7YDppC61v{3D;XJV3G70mzza#iU&`)X&mW$e}caEevBIS~F7Kc!cUv z)47J%o?n=ICPJll(w&20hFwC~jZE6#Ya)ASX#Hd0-NcT7K|$ZPwMS{WQon!DU)HRm z15|F)(?Y7R$dJMYd0vE2u6ZdCdCgMYxWEs4p-I6NYerytu$}I*+Ss^Wn4;9Yv2DxD z6b-q&IUpy=C_=2J4RdPaATYVfKRfy55#FMXe=J#Hl9yuXzq zjf|D?lc2G#7wGEinMo6ONsF^}e>;NU@h?`u?kCZU$8j+sR;*XRwe2RisIt5AE8O#a zhUtiQV{K~0m!gHj$ zM-scebLD8PyD!=FUZB?-&-bvBZ`c%h^FgyLoZu^V_mMH6>Q{UV?g6zfgMm zHNguQX^9x8enDR&^N7>BkFnQ;gpN?#0`((0>&40~su`~BLDyA#OSJlmjQyMlrjC-c zjUbOYbBnIuIHRUC9d(6BzR^#|pLZ7SN>+@c1-KD6pTHfq?6O+pnGPW%{Hh0=GSh=J z=h&!;ivkp>zr#j%)E-LLUfX@PWx2Niqb``q?th#`vUT&*>UNVUoHV;hA_k{aez!kH zMX0MX3uD(~E%Zk&+7vVj)vI)6x+p-E+DeZ{N=~T&kz^n zCg8Y0JjB;rgTHh`1Pk?swRZ!JvPE}*3pjP>CvzWK>D`1k zzoEo*5R`+MrB+o%3E6hG8mPB-B9=D(^yJ&BARClKooZD{Lx@#8z(;ufr=I>?@0FKZ z0D?Vn9nHe-hrLbb(IY+m#dJ#CH^sRj-c0w<4P%Wg4n4JAL1z(qfDK<4GF?Hf?((M zoZFaYiAR?KwQfSFq(HL?7*YK)&l<`yXaxrn#D9`$jFjQref}XCUH1F}&2ZzD&x@-7n zjMO^BzfN>lRU})8g+R3g6KV<-pne4TTwq|o$UjX?xYJE6Au!ROP`m@AVirISg$Pi) zTVK*ut8}Onm(6E6w2c6~9c$c9>EJXJfB2#Ljc$RBE*+XK!L@Hl5jeh)pt9%#9EN`voy;xm@s9d>*sSYbH+2b%XZPKO?xW=7PmaUb=LkV9;Wm5|s~HWn`wG#S=Xl#qQY%P%Buel`g`Xy~R7 zn_`Je|1LMfDDdY;>cLQ*@cN;%w_-#W8?RKDxhVlEe}&r7<^7{}tz4O%x_W!>b~1bT zEItI=&%cCk1eyXkSS5^cOhlEmkS=)(2?s`(6lD6J{mcH>`)93Qf?9-U1`0d|nZ&b~ zwg|ccrA0u*SxPd2N?Jp+b@KtFndm}LQeD9TX=BI(N_bD9w}t`+2nICU0h}Km3Tqx@ z5_c6FiCH@ZB5SRIDg)8}A55-V21znR_n2mPYC&6!%>q#*^frH*vCFE1O_;xT?~!IT zmwirTWFOCjs}0}$}INf>DGtFMsI4VFDEWsJ;YWeYo)ILkZVmTj>G7$S0hYarK%5TxXI7?(Vse7q^_IxSL81Au53{I6?BKA&IoEXPPb&XcL%uWE( zKUES1@^gskO;QgH4Y_Urhfu%_w2)k=dYy%%5mJB(^k zrehz2!_)&2&eU%EHN-#l^2+AI{RK0>Ahxn(SkKHxxo)6;NtMpMS)08SAro7gS&8aR zLULSa(eY7ZdF&qv7hUhZF$jz7)z3e$PgnRDXe5o#6LsOr`(`S!Zk z3}@&!=d{%CP=>i(QlQ=~GVXKt2mesrws+?RP>XJqr`Z3j2cAJ`D%34Xp`|01$Y91E zyMzZ+US}AyMA?$EP+b?M&~oLW%+*&vq%ANN8t^0uIZv+t2F$#e)&%9Y*GRpfG!W%{ zYe_4Hd(`xJf6=DlIjh^!u2;-F;cEVGz6=~+-~=UOSQD$<&qG;>y%8=}`$lR9(D-)9 z`G)Eg*G7RdmiLhJ5+oBtT|LJop~n{AyonW1TTw1vc04JmQyA6BIUT8h{xF1B<@6_P{!9BIgA^beWcFmTE+jI6d>guf1kA)n`=*?Bmi6}!1lTlHqGjDkW zK%s{Vm~yf$ywihpqt*MRDV~MqV6J?ANHPPHKvP|jC<#F9l@Kbk+>Zp$&7r19K~4EN@wO07zuA5Mg3G%-lk+^-)ZP zt8$+vL`qL0b&;U6b9eo!?;Dl1YDGsF4FNun$m)*fzDJ*9^Gvzv7KY^SD%o*R6c zEH=Ul$VKK@!lsH@5=adW0HsQaLq1|8YL`ax+a%I}7EzRmL*c1})r^E~iy}L@@LqoB z7hGD2E$*2%?0wv0K8HMHE$8sT*TulhL)j)341!aJ^tjaF- zMF6cdY_Jq+13MA4qns5GShV2J+4|-NPMU`Uv@92pZ=9sZ%R^tMP^?35aNUMJY8y#h zCT5yhL_$0{?TJlS_YI!E+4t{7wZg^Pg)e^@Dn%ZM;KH}{mW>Hn^{g3bo(R2t%KE3o zHE<*JNwQ+MP8+#S!n!Km{dr!R(O|+m6?{Tq!};wjHn}gFW;FCf2%Y2ve5gW|DODH~ z0Vf6;QJ(5I)t^aaV~wM8rPeSe27_WwZqDxhAU=+h@k)5S3h0kS_R%wc#EvMvpHFR< zUa^%=q7}-jTimkCcUSJB(_j~#-d`N;I>2LFJ$uH16F;4(x(%7Ki%_+fMWdquiV;q+ zz`vMNdMik}`c3^De&rI(mhg5+F4NlGcfem&MbI$R&59ILqauvRNfPPJ22>%{C&;mM zZjs2vG73;S5WIbKPhCWUC*~MR?7%eXli|bTf~5;KLsAP;ed9Urh-P4C2zo*AbJ)}j z4a>Q-$R^R^+`^H@vK;H*rH-Ie4hF+6v&ieS5y<8+9(p%4G$_Jjha#78AP!Les%{6Z zhKXOlea=E)C{~d2m5!#B|0YogQs1q*LNd3w8(R5+VC2lEuAW_0DfWnPcW=5 zG4%llC=OE1;vqJI{Ofhx>;=TS!+l*D182|{Lqu%KrQMjfI%52r%TPNM3jB|U-gpnR z-GQVsdODAs_6SbmJ%iJ`6+U2_(7?a|LF6nC6Oo)9CiUa6$34%e z;q|hy5C*WL6G9lkl9>%DRzct+j0a1#bcc-Sbzi|R5Nd~PGC`rt$6KtvP0S1yJUX8) zaMhS-dDfUHTe_$>D%|4d}7 z9Z4PqEMD0L6s?YsLUi8M?@$sGb-64Fzod!hdpKp_%?mVXZ#3jY$a|FQ2UH02uz(4j zJP2Z1QJo^J*zR(+w;J}nS5?}6&^sWz!svS6ajJ@&!N=i8GkD!^1p?o`S`Sl4V%dz& z`#r?usFGc@boPjD@FsY06h4tW-kyd2io43r1T~roFu}3$2velI?u&wa?NaE6jRucC z>V1olf((;xuet4&V8y+;7PU^!2H9y0{IBOe*49!FXA#-EX(I_q!iq(3yV{(W%ugFk zUKfhnQjQU}Ca@jO*Ok#mFkcgFk4chvo>NR(;JVdhf>aLbhN|Byk(O?Rkg!+^&Jf5X z}!_ zO=4BbznA=1ei)r~d(l(uT&S%D@Hko~C2>~2TxY{bj1A2j{&Zt+ivq@ z^xmi055S374eKb0&0^Ye3QK;X7^0=tLmQX00DHlY`>#;V1zC7fMF zo+c>@tl8vk!t>n(u8tD7wOxp#|yJ(RToKCEQUS?JEyr5IU8*QCLTh=?H>+a@2(-+w2&m*9o3kPU9~Nm4y}M+MbB&v@(6S zDmVv?^(5;_IM@jW0=&CbgAwxe8WUhztefG!(}>_uZby{S@No`8m|y{i_Fl^6o0Mb? z>k4Bx>(5S8kTMpBp%Pv_Hss;jt}?MstuR&r0>vO?yAkoE&PIy}DPtfB+Tv1%gNnqd znYpIbVZ2W2<}HFLra1+3gA$K35g;t)B+2wY0Bt~$zty(;fLLiHsWzi!_GnBVrfa2L z`|ZyEQ9|{%g^APiymGmrrHk|j33g`^Q7hWn1Y=486`dDpBZI2rA{QWRdnhYyjT&6y zShQD9l)y(5ZDy!MYiaWGLw~cZE^#7JL)T}( zl={}ZYKGF-%#`zJD6$U0%pz*K8QE4ffvx^}ct$s^WRI)Xk?bQENx zzwqpsmWULiBPIr=0VIQyuXk%utUfPhoKuehXD6wi!*J1STvVRja6U7@m-vm1{<}U|$Hf8A1Q>8DEz=V(}jf{b?Smz+?|A z2oe=DdSGH85%^-&Hj^YDR-AL!RL%fUaT|L`A*xu z?JiX79UVPn1EvLG7-*?R)#v|86%>+b^deL@y)7Qyu-(yY?{*%Y7e}+kchDf4%2SWH=e|UZm7k(&NoJQsk8{$a%|a#oOulpAa_EP9IE!H zC_w~@R>2dRCAAV_bQp(f+-s#Y5_TrUO}#aH+IUWRf#ERictK?q9;3z$+s#p-#rNB7 z#X>+`-KLGT^NHT;Qq*wYOo`x8OW|zR1Sr|F#C4N2Zw#0#Q7(C!V#OO6fG=}sDXITdj^6Qbprcr}=Sr-t5L!t>%nq*omk*}d;p_MNV8 z3>keaB`4;GWn*tUb>`6pSbGGc2{S@hrjnOb-Cy3YyC@oU{0?94r6{CqT{0|Y1lJ3$ zTkhf3OGfH~-7>Z9DVMtHu!)(tRwwtnLeh%M6*y4;=WFp^RK_3TEF=Rd3cWPpt)GGl+5gV*ZmT7)i2P zR=!*kbBv()E4UP%qoe>U41~WjbhSeW+-5Pf0#Zp{Jbm%>`AemNpihWOja#*=JW=w(q~Ck(@YKaB;lcMQ7?z#IT*Bvt2Xw&a7e7wAewHZhBxy!CA4pqq`tMFmlV9-!ih{>kTXI zXj|}y!#(CEzZN@lQ!rjF$a!^qEWx0TT9ZyhFdnwHq~miq+t0MS#Vuj!3_@3syXDjP z;>E!}jl8+Rkz#4{zU4uu160Zo6zBX>LpXleuznfsJ~h&1Abo)Z#PTPgT->2l?L23Y zd2P5o39x(cOG)zas7o(C9)YK-QD}6Cr4}18YS!7NxfZ&VCP+y|+C~f?K$Sb|(_07j zubO63k^VI}*~+~Y@jDsu;C8_jMPbpG1v$fmFTum8=HXREJp1h2GeZ;HH2)3+SMwgp z1!UH%W;7wXP}~H3=SrV#j^DX^qnayQ@N8xRuC(WbEOlHM1rL*bFZ7aoj;Z1`Xh>`n zVP>wW>~7ODeL>*Gkn0x4`9q&RHs-93+yqkeR~{znW#$7+`^C&!1>Z>Z`+tn$@hva+ zGrX6YDMvDC{J&QoYme#R>HOnKJKIx;crZl5ZI)_?2Vjr39`8jXph#ZTKtN`sV0XY3 z)#7;X)4-zXq*$Ao&Xn8$_1tyFu$$;fwXsH%`{LvaGAGYY)hRFV8o?&oqKRVBUZfZ` zP^A>89g&be1I_n@hpu90DR6Ekxu@7QdorNZb(A;X^*mImdmNrzh?JeJR=nWV?m}cd ziHp=adcXIb#iKXkj@&U^b~t*@`CouX-5$hK^^fLk0iSAEot9b+%L$3(61Mr&DGUqV7mMLB&S)H=fsAm>qmP$N52kXSkLtAfb8(F5u8}9U@ zn@~n7O1`mrrx-9$bFYsov`x)qI*Sw zN6dH4RvC5Wq(xXb)XH8#2*=nt<4q*6*_qY!pYv19qjys+lMcqYR>E339 zW&#Y6iXp0Bu@qnQ8Y0`6|BfoDWHxHW8_hcB)NPIEUJ$bo5l7$MB&OmWa?IV=2F7yG zId_|)Fq-TaJ?G{ueG?gxd)2;gj6^yQ9d-*1u?v_3s;|!3Dgl;I?Mu)y7?(|e0#C8J zTxLtDN&@C7D}5=1mVO?aY24)TOf=k8w?%C4*tV^an!aRE$iflRmO{iVB*`l zB1@XlB2gEmA7xVwV-BkaI>#1aRG61dsIinmmsSia2aMi?`4l^9P4q02Vy4KAX?Wil z2VLOLZmkBC70nbGt8}Wq<@wV4-edfJdgeqdBjJEH{VV=RNyW zoH2c0ZR>I_h&7}7p1Qprith6jpz{bOi{s09lz6~FJ1y#& zG1;-rpBE-^S(K@cA=jJeY*x(-3wbEMo*RA(*U!4KcLK-_xDi;3NNe}^C7KGw3pW(BOa zpj&4E8i2Ye$|k$3r;g0^`X|0UU~SZAhb(}p{S-0#H~Li!6w)4fJP43kbJ-+MY)f&= zQ>RM93sZFC&q@aVkC%T|t1y_T3P#?ytoZ-f`~L67Z7bjXxqk)D^qpf*YssGJY}3x! zZH{AmQg7#r*K(5XNxB(`ge24?p%9=w%FFxT-@&~A2}&dgO7e`SA%18l76}3uz`Ymu z+h?v8FUhlQCb(cUW^HLCeIgb+71i}TPZN3tv54jp2PcV2J1DJrh=_Xvqd=g~udmMZ zxdE^)TWZvY3DO#GXMrvQR{L^BVnnx$tyBYMV1bz+eoptLIb_D7cBIR$nhjFxK|Gtk z%4yVIro>YaHKY`3KoQh6=t-NDn2RZ$@`S4uq2S}jvU;qv=)?HijWwoX9ie2F2#SU5 z$O%!4Tom&KdDdAL|1s2J-J%Ms3I-y0QwJSjM$oWzOaZhpP}N$C0^j?*a0XN&gG*fA zyq*{!b&KW=)>O@m!B34sxq15CxB)FiQ8z|$=AhitLM=qD&jzqI7!D#Z>H#5$V{5O( z1eD0}FXPOk#IHYO)zdYO7sNowhpI<%C3Ftne?G=GZV?8?) z)o^wSTGl4|e^u?wGo!vdEMU{=(S&N6c;$4oMDS5!8Zs)Iv4D!l@Zd;mqv&I8zm1YX zp$wSEpip*=?pMdz2ZeG_C;VJu$XR&B0U$F)xUd2f* z#61xS7WFICNX$k;_jhdmV=9WQHHHzL={9v8!XI)bm!^(5Kv+%AzalC9091e!x6|#v z1E@)JCKcovxnH1?Kd2Q%%NyGl?fkWK` z46g0*w@e}DRxM#`?5$4qA{29LOS~-yQ>%4yeXzD_cv`JJ_Q2VyV`;Va*nzRtoy_V{ zEBA)0)j5hTJgo=8(%KnEtNUKjfuHq2*jf9=&FWmf0KBX%v9i3@j4g4pe5o&6Vq|#{ z{Im;{mcQb`ylTa}+A=8xr{AU9AX5s}YHX)XS2^6BqEO31)pSoV zKR^fL@^c$#q76N6YC`7g-8zk{!8DE4=rlX2H1!(gdhKF+!XR@7p}PQ%pxGdzmAxop zQey1}nynn+X%72RG=_S2;#=Iyi-bzWqpuPX-D+wCX0kTDS(|A_PQROwb1rW+xqUzq zCRKM0u8fk~V#*Q*S!19yq%>M!mbEFz#Xc;B^yjEs+VnLhO57L6W+(lY-b;qlo$Bf8 zpIMv9FU0CQlkM3~P2|SsZ7Be_g8>*0oNH@=34nP8O0Psg?kVQX0H+y~8@_x=E;3nQ zK`piRnNE`+q=c4N_?%X8cwe z%km*_m(!K~y5_*nCWM-%=BMMB-5txAK0AUT;7;lAnbzh>C3*7f$(T&6estu{yL`^u z<4ok`K%g$d$rExYau`uWGm15x43v9-JDj2yuw+w#Ca>=FA zEFCYk~7hg@xu%3P*qV0$WYboT_yFIQW`g4&pt(OETI_1;r z%8mXzIj4%EjsqOXm$qLDM6JF|0j~;Pag>xD^oa=A{``%IdngsoOO|E-Rgdrt|4vff z8+j`-=8xC8W;7=HNpJKK#d>EpiQSN8>Jr&hTPkptfqgJBa&1uh$IvTxrTHz-ys=#MQ>>o`$6HiYJb+8@WI;6}C25f;zVsCRA$H5_4F)BP?e5VR zPvBDIoMD}!_@#xY>D+;^T^l=lgTRM|yOubJdXsnec!*}otbENli7*>lGWB$9wL`>B z0W$2&je4s*rN*;i10{$0w_~uaJ{<$k?9_V}EZ|IyTBOy$f@B)(6?S=Uv;`b76F{5-~8+wc6!JV;m^c#umXqFeSj+hw?AUXxa}d1 z*m5xogC5Y2zM9l6re|!qAg?B+hV6I}eRc#GwnY2tfg}l^F%N)u5ZtQBxDWhyV>So& z4ZXV8b}h&o=dYARB1y1NiFfyk84{xGvf=7xinqz}?VsTCX0B<+xB$o37pLcMF2-s6 z+l9CvDKRP}8*$~6i~-n$fuW5dh<;0tdyh1jo76&+@SP2s#8^EO8kJhdrVz@1d&#K@ zoTSG5RwN11TIGx#xt6BmkFP~khLhSl*a9Z!thV&Zg`mZTvCr>$I-ksZ)DW z`+L{zacP3(Z`q0fH0Vs^dT-|Sy~I=mL^v)a%g6ZX$3omLK?j4qW-X~Ln^)uNi-7UP6AXhugV9-vQo8b7h$wzKk0-I7!m2C3@; z<)0Dr6GL2=VhKp#tt3E98((-;$k{bKyq}ZSd8y8_QvH`HS0%&oy91d*U}Qx6*P2aa z!0AA!JJ4t2syb|kO)63rH>Si_J~=r#LG05nzW9QOoKO<8h^OlhdOOA*{{7jrG5Md< zH?JZ5aj7I?r{h=k1GSDl2$#D3*W}&X3PrB|CfpiJa@GQe{lom@ix*>Z&U3&r;0EZ) zMr7KM;Y=t(0Z|jwB>&xZfzS#H5H~SRfX^4oVD0GLoA5|s3L>&j(Sb)rY}6l)p;QB5 z!bc?Gw~Wl>*SRQ`CDN%0koERyk|A*r)n1fLeZ)AVPWxo-EWy)j8leG}xvf>k_Tl#v z6UzaiADCBC7pxJBow7tWU2B<(f5H~KQ^T-CELK!Xeodpk%ikaTTepcfQ2ERAd`HBV?uDfQI}U21;R`ycP)#$qWV!)z)1L zSg~?ZYC(CDslqH1gkkD2Gbs^9+S_MJ^ILsKG#q&hStqir^OCcKDt?FFrIfDRY_86+ z9*oBa$U;O;q@uYw^ZM^T9U#D9zgFdXska!4c3WJH)^bUxq12}<>2zBML4#_0us8Qw zh$-z5%MAlNNR3o}x206Z)*zb|X00X66~?Z>4d`~SA{pBQ2xe@3#53j{htRdo2k%Qw>t zWpw*07B+&p2DpYhZ`vWQ;VzTM<285e-P94Z(jk?EYBaMi*rbQ;t89X*7hdcx#L^eA zk@}l!BFV!n36hh;vohh%1`u6lisdsJv8S$ufh`v-Ww{3*-UdV3oVNwdVuQ*Yu^bo) z`=D5^*Wzq)N#dN}v0ReFuY9TNrq)WU=Re3i0bdFaREEbbYtEb_R}C9yiBs&lqLvnx zvEFZxpb4ySC5qhmyllaxNaXs?-IfSjHK_VA*H@6-!R$sPyq&{zw+oG~TrTa=$lf`b zvMUzO3UwQBaaVv{ODc^u5nSb6fKJ$)Wy32LXh&t$hg#ZIXlkG!bh%8t&i5c%XCh`- z#Wdk^G3jmeZ+aPnugWWX7Y;Qky$ubvf5|e*K`&XWxTBY$i$3cIMnFzq!c3hW{nOiO z{NHw`cxqiyzwu4Bbl)KnO%l0Y55uy3zmBfEge&aS?li79p`{3fZlG2%Mg(xy--#kg zDlqvpBO+^$~$yqp$A3#zuy~k#o!fEY#(}-DHfFQULk@ z{?Svn1qLjl0&39#Kd$3qqG_RfWxQKAAVC7gxvg3oi~H=J=JDxOYyMbWC99ryl~Vg> zqW?~(qEI%6!Q=4S^p1T>FIba-%~YOW_hfmuM5C9>XTBq{LyB$TR6|Rdx7Wd`HaOKj z52xBCI@Po!O}K=jIyHV{%TW2Pbbb96=M|?z^>#h&%hANUG#OzIZMBMPmBE+7fR*pb z0|I?@TwogJd{Cv3AbvhN2OSUOQgoR0uz|8evrM35lt&24FOTpRulFlxmP;BjZ#Ax} zm4Y?Eyc|v_0@a4$E z7hjuq+aBsFzxgF_#L|s>&4QI!{wO z-d=7=9N%71nQWb+S6eDYl`2$LJTen5izEi?M79S>5}tA>9uNcLVNywQh<_YM%OZql zn5R>gihOlccB5af^=qDZ8p1DgMxr9mS*DWJ1L|BQ<+_sDt5#o>%3TKL(^94bFE(jO zAv7e8D-+T7vpaG}bJ;y((qGMr`5l)cXuD{MxOx|`SqbPVeVAPRo#4SY;=*uKf8SJF zMb2!_MhcW!)cTc~3`iuRK-#>kNc=-L_ z-h6#?b@31W^q23mBE7}``sMI@cJYtEPM$vfvKh!>z11{6o~e<@M@agYP^&JW6HB0N z&PyevN+won=?}2g`>5ER#artmcyJ9nG=AmdMW=Lj1TM349+<*tpROk^D3&foPV*Is zjcIJvqyB|c0AuH(e=xq7=euMvrYz<~+M|#LJnvHE9*WeO;l7z z)LTe@5w$-sizd=<@TSXU!Xqe%4>wg<=5pKIKuzBdJS|dENWIBLW}fl(YEDc!zvX0oLTwyZ$^EUP_@*@0 z7_k?vB2&4A2UUy5VZ;Ulk1I0L|6`3k*26btGYnX1KQ+29xL=!en3yF@K?Vye%l2dR zOe*g5=KLWe-$-<{w!3F$FA05G%MO)zea#q)F+V0@W%qfJ$s;m-YC{CuI~FsN=?g-M z-hfu5oM{3>9h+RFL}qj;7XsiVBf6v!R~{V4oJP0$Rq)xt@Qz6I)Py&?%T`bEQHxv@ z^LG7>mTyN1m6p1V{hC{{B=IVv-Ko16S0}kJ-X9nTq2#J8y^Q{*vxM@L97@K>h27Hl z%fzoPqct+G$BKy(J}SdnU=CC6^0}J={F(xdgM3=H zrF)ylGh+%=dnccWZ0^?Es!pgD2N~g+zK6#JO;8Zv;9I*qn_ObIe7;bldj_6el<5_( zv4iC60jlk%OoE_M#?FVi9)!gyE>N8EVctjJu z?XJE(>xvX8Piid`>&CLxtVBK1idwHTN^k?Pbjsqm!k$4>%6-Oidn&HJJv$C*x>WL$BQvxnaVpVY+Z@i0WZff=j@+0dIFZLwA)5hv8*#-*=ECL zLKz(E8Mvo3O%B0Yh{X?%h!O($4)8*6pQjmNAGm}{C`ema8j8W@rGlbAsds`ceBV zvzQc_;{NC({bu=nAcL!R!-HBFDY0mdx}dD1F)NDn_n$`p;MqP-%s-_Q~{ z%S5~Y!JJ5#u$;~ECLosFg=hwTv9L6PT1sk6-ikm^D+XkPsthP=Sfi}`8Zt{3lehqF zyke$ZKEYtitk&*&QS_!t{Ew^jN24bz%u z&zX`{a3;z#%@~dLVk|cx>|hx>d!@!$(R?w$MU=wGB`tlsCf~%VNTZ3+^ zUapcaFu6y9c?%y1ayz8Zi(1FHB@24Dy`{O*UPmR;3=C-mpBp_3aft*(6x!C)(-xAi zYMbU!lVG>dTIHYeUBl{4+(sE06aY_U8{?V=WA{OQTK$YltR`vW62R^?rgqtymh%3MpNPRbj@bQQse%q-hWW9mF%tckHWXs`M3;Z4eHov_D;P>lg?Jf z!@72=un-2dF)FtbPUB1hB0BoUbU-96#Dj%By~d4BuP)(bdoP^Be^_ZB>?F2r$}~Hu zDkcG;-EZ^`(Zg0h!+!(Z4&!($A=-1MzZBdh%ybMefizp0AU(CVXb5xE# zxMMeMGVrC|F(F-0LT;}0wkC7H-ZCRx99-Wa{xMi8S-fo5Tvol)9gv~KsOSC@uKA7&y$wujG#DQ;*LJr^T|UctW%cBsz*GFwLAc^2!;$p`UDq1i}qzKyXgz!u% znj{rgx7h#!G*077auv}q7Vc!<8w=<%p~gftE3ybf9l2Wd3DVoRD1cQ%qLq%%(34PE z7s6Po-AG`WTcH$PSmar_D4S}6uS8fxpJSwQumdt!~m;^5Y}m-L`oHpzyio;j(06BQ7UXKMc5GqRRokJ zvsmU#HfyVa`+l8}&hEmcacB^>N+q4J6r$0`%z$q&gakb4ISV|TslL=vfWe*Fnh>SS z)EV%`LWXGxza!s)58der&8tZw?j=$8qO^LU_7}EXGsBJK(b{#u^sCiNy_8WGj(FK4 zba$lB`aMe${r^6W-C_Fn%%0CP%Mgdva=L*-Z~%PSiWV9^Sy^g2GTw z$u|Z9zLYOzUUrLpAWR8D8v@z~&~sx<1Y>e|nk*O2mjeY$$(zd5)jCz&aR>EymJ2E0 z+HTv-Aa)9&1_nFL;U+%Z#G9r!+{ByOui+-{@UX3?T54x?!$KkhO z3@N6nko8(C5d<{s(ZyY^{X=M+n2ZrBuKJgdtGykb8B4azjwRb=$dbpJoKbIkNJb?a zyyrD$wBsMA=2~boi>MzDf4zwr*zgL)oRoRP6M7$sTGhqllxB3!MrF67DmloDzU;Q1 z&I(ed;t%KJ*Lyy$ZJCF)6M~XCd~wUX!p##>E0Sx!*GDH8DOZZwFz51QKzvFZSYOX= zh9`gn1>>68hZ1Ykxd6aT`<{EUFxHz^wKKxgB|;&<*&dl3JnJ~ijiK7rR`=`mPU^)8ffT`8t|X(Yv0|`jO@|G2tLb+7_B{L^)gz3NY*-4-x;ITC zxJQEIEBgRWQHG6{)_QWwR^i4$Vgj!XRIu$9GY)F-W<9#>F6moueK#kzAowb`wLw|o z+5i?!7*LSgu*AAq638IeWE0_&C)?P)u-P@+fehuc*OGv5*{Y+W^Jc@URw9!NzO?YY zaL&zcAm7r2_t{164DqsinH`b0Lh1jzfDDqJ`%<@a!Q@+^;Qf!XNI3l+mdN<`IQZ2t z%L^)P^c+VumNJ&ITzoFcWky79sJq1xFoVS5S@%Na4e!}w!}!dCi@dgidmC!$1Mf)n z*=*Pk*iRvXq)ZC|Fh8o;JRs1j-cpv&p*ma?1bEqmNw^-9n%?$^PT1QHkn(O>utz5tX`hPv{u{@V!a!Mr6 z<_UBAux35ybHUx}5$Sud(*BVsK!ovmQZx3^tbh|_s{^8 z6-1|r(@*6rGnum+M(Cg}cQjN($&IiitX>RZ9a3Zm;Hh;AbhdT1~UfV8$@ zdx|WJd?}>CeOf;=VVskf6^(6rC}u5}2Lw_`P1fSu_(OsCOGm zir2Oh>jsC4iRA|e7Y+PTU!{~A1ExfMB9IQ>jcje3WC+6T;H8(cq+c-^w?RFw zsVFkD?`@T@-6^VPZ)*ya!!xI$N(>AeLFI7=*SO0W=0Ma*X=f^gzELyxeSgsBJN7GVzR z){=LBPE`E=is41UE3qEU*c?Z>{T}s1Qa+F*?Lt_xtpV2ys@vNFXth$C0R?1i=eHo~R@zAQs?`B)zDpTe6w2V6 z7%wDNlzTP`tFzE+_JNvr$R&FOtK)ef;9H%K%vy_B)B#zey7}=HpXQ95Eohdp#3{s# zuCdlJQ>AFMKv_VE=35C<2a+s=R3xR*f@fuk%&aV@-H|`B6s@XCh}pU)H0^=GrZwBM z0(Zq@UqRNs=_>OAaBA!CSEzf24FGBIIt#)Hp7n3qcCcJ>nU+3Fo;h-vnHRkM@|t!t zHei_xaZh4F?imCkX+0G(3uFjljx&2m%8Nw0%tgv=l)T*&I~U+h3NIv5eT9=E)5}{S z&Ad7ziso~sbU!33(kb&6HQg`9Sa0aggA%*5HjC|nHwn46JwgT&=#Vp!M>fRMLWLr5 zoL>2FjchD3CA~sB4Ac|J|J`?Q!@<@Ls(0E}^Iqghe9zW-fJW)fi9?S%yE zH(uV~^SB2w;PE;U?A2EjfoLia9ELksffd)!GBbh&Nu|B$5Hm|kMHBP{P^i!%QqCe}+e)fvVP#Oo2H_8gId$6t0?#d@hcoyLo8+=UMf|2~6)y|Ak*2nko&A6@u6wv!e#dPhC&`N}YhSb=)r=}Cnd&mf z)%COL9nC%5Gh%)M{}YP|?O++E_AD9w#i)5%t(Td|OTt*VAwU1M;kEE>P2-$J9o3J6 zcE1_XH{V>I_a&2l$0~yG?>eahJbvKj;Xf7b6%0!Jm&7rB?KaEKQ#6lQWimbI@-_^H zYvZ;0(fa!Af|$Ps-dvjPgk6Th_HIR0F@8r(n|Z`4S4zy~?Jm$>+nXNO?+X@rqJoC; z?zfdF$h|2rZtf{JeEE`GWU|Ov1q!tGnXWw?cxxdSSZiLswD%Kl$30D$>t;%C-}~MT z-j4;@OV@3pUAK&fv6n7suHd>AzB$WREGu;WknOrVFK0HPr|t*mvV6#Y$u4WiX({CH zz*i+M3ptM2-LZ`6vm+QnNVaZ$o>Y=2&z_9Qgr_{Ad4dJO%BpDgIloC<g3fX_Q3)ip0&ombAPj%&%+XBT7g zjk(Ggv5Z_z&tLCpc;aIcHt6W?U?B*(&--P-+FgIs=kO^A=C>zw-%M~vZLL#rbmK}Zfzrc@D%%p@uj>hl|+^><9pFRrdH&Q5PG&R>$R%@??wZxpG7~iO~5}{T)S8r=d6#rJaU|9KrGo)ckmS zFov9rgYkh?#1b1U>Fw0WrfM)Qep7wpjR|>YlUw~78{df6EW<1-2+M{qbnzCjTNP7x zuqJGC&hJbu$DM7_)o_BB*g2f`8hHJB_i^B9>@f-h@nyyki8MttnnhXiEUds~$vJkL zLpNvbT>5VAiYP#{)+4*dU+v>-H`MaaCMDe;x&YT2;8(qnqYcECwDq8jNINbEb$oqs zdj95OoW{GbD3--b!83n}hBX{*`-DzOxam^WiE_aMY?F&KG7 zN1DRBUu)E&&y7YcEJ3Q7mD;>o1ZNX`@`>+RDO{&Umw#i*@L&NxpjtR6TuYXx9CLqa z+M@rC)T&879D#RLBFiJzu4cBQ@OQF7&5^D=o4dZY_USqkT;^<4qR-5D!k~J$m9_1@ z+pIG6Sj(KXDw#Iq#`8I49@Qab#n)|unv zDfy0>??ZZn%6X*7{XsW7+h6@{aUcyH$IW8H(+KT$l^`r%ssA+R2Lk9X3qx8Ts$ z+Fojf&W;?1KqXl|Pl;Tm3D0gT5TlteMG_g&KiK`7GwRPTSeFhv?SP%Rzv^XW27|Ib zXMeS9!UTJ`m-#18V0IlE`zM}xz}0>Sp37w2SE@zt^R(YXt2LhU3X8Kmr zRk*nx|9nI;c5lYS?fc|Ej9C%TK9aw@42CbSpb9PGtFaYDBFk9R1)3YSBtM0-J^oX+ zW+}y~{&ztWB;VV8nEC4D4&k^|%9JRpZE zT6+C^d(VMMip*74aGW=|9Iwk97~hcg>;!~e9d6Mw&zI}tj{gdNBd@uH(3~zjdU%*i z0}Bh9+m*g_^5u^-j`No&gJ(vQ$~zOrEffRIN-y@IS}hIM(aqTv^diZh{&)hK+#jC* z@x;6MA2;u$wjE{HL~|jHGy9klZ4LSPJVoS`qjm2pc_WFcr21?VYKpX;vU0KZQJ?kT(&IKX_u|J z>oW58)uiUB+wjP>TTs?Lb-P}jvRNwy9?0J5)Ss?hQZ-Qk94;R}q;1YKVe*OGQF`V` zjV{koe@9feVzF~@T@XdP6_>nE2Ic`iZ*>-ZevtX(n*D3W&!h~gy|%vVKIzWd$>2LO z^VpWnFPlmcg>ksXS4{~aTSckjf1YDyGT3ZEqo&$OHjaj-4w_c`{@!1`q4hFTP<~}6 z<_SIYiw-ScSx-K0Ay?OhWRsMHN-K3u_?nxg{1BXZXIhg7BYL!{tBKr>n_+i22V-}R_2;~t4@`e94+%X6b)+DtTQHW>-nWn^=wJ-9@VFVD9=m?BsABJWeAqyEZT z1daJ^J&CmaHuAX6-UQCx8s1*p8T4z!<)yXf5%l{(O26CM5qw-nZ+j`(QI(+-o8H-M zRh41q`v-3M9z4gsUbpzZPGi!Z()01y_lh^OnF z5Pa(2pFJCs|2ci*3+X2YE>~j3DjX3kh}*F|BL6jc_qM8&YJT;0Y%0;~JyMc$OwOI~ zJJ;4n)uIz&3CbnUW=?5aE-h!U1xNUNVFAeX9!q#fC3!HXpk3J3yumO__)aNFUXpad4jiPIqZ9u}mp$@HOj6U^}(D@wDMeagYDbtK1;XhY33>S=CUA%!9 za8QZ0DyCS5$&Fz(!Sx2L)Zj587g8vw=%;%&3na_J;1EnQnkFLhmlMDiiPT_qEBnJXt`wZn27JYFCP zxE|q@tDNR5R9@`bX=a7^1c4NWQ6X9*WMBE!ebO7y4))ok2fpMQf5KNm#oIhwvJa>o z0Op{;Uf=u#{UcNo1~NCWM{bW^8K=h7Sl{4OL6c-PVjnbn=65VaSlEUstp*Q)sUzIf z%bY2!+w%nHkqOAicmj_Gb`8J_8dw&Qio!{Ha)mU-6v~el1Kl7voSq)!R2a2<6W7j`D;~6MN5xP$#tU zLGrN;u4?*N=(uHBp&=8%YoI=b`@w3>Lm5puxJLWqTVsB#?~DH9Rl=BRerCC7GWzt{ zYF)n9s&$1ESFN_suHoi!XWT~s#pG1&%7qAq-4haVvTdm*W>*d z&u&bi6s&SGGlljIAZ5eA1ADjWo<&xHl-BIL`(LkcrPLdsyZ-ec8Hf(@H+0$4rBuG> z;y4gG6@{|dvG{ucxu62N64v{+XXA8_t4TeT6+#c{u`fr2(P*Jnun{OZfqP0ToK~|u z*OM)MW$SXM2i1tPS)G6h<98`ztfUDt3hfbe%*=@Zwl+9(*6g2Q{-Ju@CPH<>r5^4r zTglB?gvt{;*D7^f9}g4iQDn|H`p^?_wIGFGGE-ccTDzW;q`u%$D97Rm%xikIJE2_c zqtK!bRg+8=OB^ng7|^q2QX(KTNojt|;!?8;U8O5cywW_8TcHP0cyv`Ouo-U1Js6J< zP;NF9lg}@Rkpg62rggfN0(EuDRw{2s8?rD2&vpbu4n$y(B zlC#xCJq^SLZXEUj#HiFjhsvmxUIw9-_$f?)Gl&5Fj9UHxsxnBbSD!%vRlRyFgReRQ zthTTuw*+H30av{`b%L`lXTVGvi{SR^(~YSs6*q~_cqdA$pI@ws=<0~t>ea0;!fP#x zt5?6Ch_4VZaN9ydz&5586cV=H!tY_&>d~Q2-O;>uGwbCMc9sNgqf3W*j>#J%@Oo|@ zs&!x~l0uawP`8$x*kEGTfQXf>ZI_ia?LBQH62pbCd9}ZQ=xcHBV{H$-4q@BUk{D`T z-_i`&Swtdmb|1Oa1h&2fCjz?-DNBLv3S4R=A6FewX_@U6f>!<|AVe;63e1766W|Bhlu>#fe+udVbmR#mjdgpV4V zu;pdqm*9*eG|Pm-V3;E`#rJX~k}{0g&CAZ6?6Y3FI@m3-JKXvOqlsU)KXOKERC zOSe~gWv+~RSw%7v#qv}VG3y`v;byEoxqB8EZCSOg10-g4Dm#(($|)d?HcPl-Cw(tJ z#Hj{QTF^AJzr(frNMzYA`lBReZ$-IhILGe3Xl%P#WEw0n*1FPbm3)?%BdhlGy#aUO*2*P-GI6l>&-ZGN~4U(YXDmZ2jlTUpVcX`3(U2x zP_xmW2+JZ7+Z>=7&wCD1E9sRDz_n-rn=EZT{Z9dJ1PFX7Q)lu1@{;{WEt43pbgJ>fX z6?x7wm8?FpW?s@SJcOib^#v`F4behY>R`R~$61L)D~~F3jQ%Uj?#La@z4e=#+SbZ- z%rp{QML@f|DAcl0MppG+O2g_b{ziE^kIMT} z?=JQI>RESQW4$e@QY_D2lK(yY;rG9co_=|F`2ES~&;R)S;SXc@$M2qg`Sch2AHRS4 z^ziWezrFeT=IY`f{OK>>XGMC8|Mkn^_w3>yft@^k`sM%WIndg@fM;qX@)0iSONfB% z)tx2xZs)oZ{S`b*zY79^OM+(iAL-z3cm3-`9`C%OIgcegwi7fO%LvFuAv8#BJt zVqP{s##Du*JU%+h@~4PZ!xHX+y$*qDpK+iXu5`^b0P$yi#u^Z%hVf^CorXrKfr~qM z>km{4uh*H}_lp0>z8e-J~Cui`vsz9}W zr8qUIH(10@qev8)x)HZ5AOK~}max$SKuoRM7D8+ZV13x^?qNgNXc2RLjo)IA*;id~ z$1G-X`9t6qGu%Y!*^bF6$yvmg+>}Y6_;=iOrtonnzvW7@WQJN1&8o7=xpOHtm%|-~ zjOq|@lzT22kqc2IG0E9KjXye+;p6;ooYB;xhJeK76#5O6F1f*;c?djx18^qMwsmG= zW8zGli6*u)v2EM7ZQHhOJNaVUw(a-b|Gxiqb)V`!wX3VT>h$T}>#Vgl4^YzJhLGXa zPZT$_j#r8x<^E-uy}3SZ&0mlB#4Pe>-Bt6IibzmL2uM_6iAcOmOJYY zbPy8ZR$fo6rHxnz0)i&46u#L_$*Gm1G^k2_oTd)pSJm}73;56%$Vx0pH6YnSsk>~B zJ5zj6tkhv7%`@3P9J9~cfJ|p<0}x9|==i`9c+*oR2N9+B!Fvot!f?&#>G#*fM1tuZ zDJF{5_a-9f0{J)-GLl2JJbnpOHkD-F?h`w>0ouPB=*+gXD^9iGz)S0Q;;s>WQL9~20S|Dr)G>Sb4@9KGAzkHEk5F4=L``a|&4 z@*`ThQ3$G}9p~Sm|0$gW&u(TlJeTtgUwVr{M}nSM-%9fL=14i|1EEQ;rML(W^!r?8#_(vP@N&vF5rl zIv+OXmt78_A`Y>G_egz{me@~8VV&8NWFz`lVfDSiipXM!7w5DH&T3Y&h~>Ad&_tnm zrX{21qItQhwIunViyxR}_*Z+pC{o|-JrSy*H0AFbPAD$gC&f`_D~FEKK@>X{AKj{0 zOtY3lKKKm)5(}Im)OH`+e+C|o>i9SH^U~}YG6S)e2zO?h$9l_5=<%N5MPpM~3+f!CypHpIQ;@BikF`VU^7WOOYUP0rYIhX0Rw-w6valtHQj<@MKs-&mzRXQEKf zCwN9Zr@}ty_+LQ`$RL7}Mb7EMyl+zA?oJ%6fM$>IEAq^Zb5QbR$(IL9x-7E>*obpq)QS%s5=yj#cnVzv}$CtaLzr=OK?DP7Pv*jnC%I7 z7Tj*b|GV{5vju;hVtpAG=fL!G=$HfEZk-*&V-o{$r(e||_9g`$LTo?LUEhW2?_){valT2KTj|?}vv|)at;=YIY}lZ)<2F%>X9&tl zgOerg#sqtCOMqeZn0M%@DXD$QpiSO&)e+AtbY31S5P*ZkZtodWG?R?hnkvTBJ&4vW zj6DR;QxIlA0&uuILP3>_Suf)zj@OnwFAhC025lh65uz~m|q&ki9`x3pOr)f=incg zodT}t8vkTVyffYM8G%ywQ&vn%CZhx6zJm3?BW^_YpDh!w3bn{3BOZM36O!lUF}uBP zBjQvQsZwxBJFJvNp{`^|bQj>Y_DcFx5cJLO7ynkFb1={3;N@4UyWRZX)Ix}z+>Ff{ z?hS2qoQ&RCTf?_Fpk1Bi4_Pkv$?DsTjq9q`+HzfCDN{N)N~KlW#5_&d6=D1nmfIMQQOTWX6~Fjl~>I1hsyoK719dJXL=J7c)MNc zNsqZ8eF?RROjT%L(-~!j>Bph33O6nL$ZxaD72#JEH@yK(l!_h3GFEBSXz93U=(;PX zTBd-1JrBmY>l?tD+YpCY&l5w|lPuR$#Lv&?N2%A$9-Sgntr_rFmy9F|o{W&44sC09 z1Dp&!pC2{ZXx;dV@m6)BKYv?sGq!^=HVq_j_}AAzCq--$U$9VJbFi(qi~JF{iFT6M za{Slgfi}E?_m9NjoH&(!ojtHxtBL5Ged-7IQ`}JAje|!rV8wY%|0<IVTE zXPXSYl#TSAakkyR82B_m*6PrY&E#NTZT6wQNE)h^Tuuu%f_q(?Q82xLUmdina^Zxmt4+hn zF!irh%C%vUwJSkf&yx3>tSiR!hk3W#J)IH^nI&@zb%Wn@h_ds8Okjkdtu}67KthyQ zA|v2;ahRa&Ns~KE@Wz&QRI{lQM>(oQ3NBk~z_05Dm%Ym&!di0q^r#1D2-a+3r7l!e z3?c*lDavr=+7CW)jkwz!7<74(C}nxKm~NC48`!s8=UV48;ySqeXynS_XxX_jJX?`Z z+bB-0Oi;ATzksbu_fnOL{s`=Ut>aUA?f#lI9oaczWH}7+cU%~4w@0n>H&4)UWN+C3 z`%dgbQXakj#8sl&&qM;I0l@+q4L^9epi>Q<+eGRUTnwRT?!SE5sX5uEYHcWAcUwyg zT6$lg%tiz0Ngaf}fZwndsW?BTzJ*;5?N{wz5i9ApMvIc0w%rvb&O~~>3aZ(+q;^WI z9)%SSK^WPr7)DEGa6jD*Qi7gTs#ue|zXL@o#_zxAL0__yc_vT%? z;1G~2Yix7%u}G(GDZ?!tQiYu4x!8}+a7)iY$dy6^h+91CR{YOQ?7zy4M<#78n+{#= zjSXZ;j&64~lxai8*FtG9qr`}g4sJT;mt}B*!}sjsXAkX}fl$x^u&3#(mB~-HZl4eG z&kiL9g#xTHQ`ltB2bmj>w>JaNte3XNxyvI=n+K2glVzl`3kRBuW#{OTXG*sR%Vq8i zyJ~|WwPZ7u4A8@eIo*ThkCjsTN4)s9lPyI$tKNR>m%D?5P%={qg#wibtFH~z=74V= z9X6v*yp|PJ4g6b?8!Tcyef2z>FfthEWSVBUgJ)SG=0;1p={}A(uWt2hTK!5#6H7c+nH9dKWmWpwT8SxpqciL!=eqXW1pNN}*p>wxaVB=}0j_4N(o$;(9F!P91BhIdoK6S?MG8lpa%^+y9n+knih! zu7B{!kt|=xsapEFP8bD@;+3Q4kd8iC^o-{(iZpVj{iLdOZ=D8RB$D*=ZLs^;(Eb=X zsFE#C40{cdp!0cC^qn&Wxw}1jG?eVtsFnPNWfsOcQVL4IPrKsYRDX{j9S^Nja5mBG z0ccp*_RabmJNVSX{Am+K@j@d{rdLXul{@t0F=?Bwow-auRn1Q{zfIZ=Hr2DMO7{}y zy=fE`Q!Z3UJ`a<~Mi0x!`W4iTorK#Wxs&)YVXt?R%wP`t%n2h(l{$ft7zX{bfpnTE zuGwb%tYOmeJX%1^t@!vcKGRin4KKaE(7|~?X->bRH0~ZkfpMcF);o**)BFB~Q@U8( z@R~DDdm>{w;n5;aq=+aENSXdG2R`&#L8ey>@YfNBxz^7Oc8|C;d{Oo2_7fvNr|^S{!oZ9X|0 z3+zII-UKn3<5dX`BXrtT>LPuAMgBU^Irgg+)4Cfc zqEuAv=wNzE=v8FR$)RP$lLjSya`&sVGYenj)x2!5PW<53CZ*+Gj)bAiw3zETJ-q5HkijUT1JvRmMXMM#yA1(gF9k5xE z_P}|#n+`q>JUPrY&O$feMJ%tx4}msSHVpfL9^J02GB_M-eL2q!&i)tDSI;(cl->;- zx^{;{F_BERxxii8(SsZG;kaRKV*S~x*E=TsBU(!de4BsNHIC$M8`)Wx3(i`X)h`#! z-p=`J8Ft&!41FBD%Shxc$cP;Y2o4__JZJmkm+Q!Kakpq``9oFqIJteD){wSXGM~nF z;o2{&e>bJB*DUrZZ1BXEZt_n+Fnn3xzNHGCgBrqv&rwUNK4ByhKyo5GfqqMLbv= z;L(Vihfg-1gjGJYO~O?sdJB~Wp_)v4fc(Z0gG143kWg^=fuyaue-@{$(X(6sj`<#Q z1665H&70KUeEaN+`4Wt;;T;=Zu5Wd{U^I@?Wy>x=H%}4E%TmJ}=piCTn6#8qP0g^) z0gqESmsuf1k1Imt1h!B|5mv4|GA}3@qfpMdv?qS(sZJgz5gE^s?8%ew*0>WcR1R3B z*SeDgm{zuXd}?NZA?tdmP@AAWpQr&}co9(?q{*dzp$+ zs3(B*(R#n>QHx7DEuUV=k&8CROh-E6O<6i_Lm>cB?9r=5H5{(y`!|<{`oB9Yx9x|d z=SvI;GZ)EvqUmh>I5WQ-FWzBXrfS#0=Wk4#x#9K4oS3EJ-ap9Tu9Q--B)%Qy)m2L@ zwiYeS+PWoMOYo(zCY&ZG`CIm=UxEEn-fJ(IbG5f*pRc65om%31Yma|-<@<0ilB=+jYY)#ew00mB z+N19}3)OQjk&c$pH0oPG)2pC{$aOL<6%6Uee}PPk*sYQnMy-NW8n(7eB%ce6?9tN? z*`gGyuDYr5GT=~qPCfjw4z}3aly0iF30SUSMO&=p zR@B?*1yQf)UWct-=!M~POlyW@aUP1KhNdlW%q`N?4BuDTrFr{~SJOTRP%y7aZ>4`- z=+5Cuty&W)rei!zq4&rAw#`bWatEBgGsj0Q+IUhvf0{Pi?AySvSG=~p!FX}Nmsm0E ziX%J#OVp6w#CA!RBsu+6Q+lE%JBG&FE;&Hpc<#t@^#KnHp~rH{euyavWzKQ7=(OQ^ zT$k>AjdaPnijxB@f33K#bi~wuBeCLbPwK_^()E~m!O9Fv(2eZl*WN|Hr|7G%?O2Y0 z2&;L4C+>&IiROtT)IEV!6emxg-L7D*iaf6E##Q14EHC3Td9}x+ztg|=d|$JEKhZAb zgHI-LMc-k5nJE6jJre3aa21+2T(-#sZl1T--_hUSA|Kxg$Fw)!GTB|Ow|fWM*R$kJS92stJeWRI)-`{DC z+sWODnOiZlIr5^(8SQjC>OjkIsPl@j&RZK7vVGd@7#S)6&|I#7JSGOdkb3%PU-Upy z^fL)R!+Fa_xM0?XvgKQXKPA&kmMS1I0A=ub)yCQly@$jENSf$k$pouc?K8c{vFKg6 z2sBRX{f9kZPUL7wN`*hE#n6Q?!j2iUjDH$5HK$!tB$eyg5P|Bx0SeJwd! z?`lx9sBgT&UMyf07rHo5$ALM4g_wY=iO^Iw{w9U(N&TS?3AI7b_ubuSYqT>t!KZ+B zANj5Mr1i6aRIv^o$4^=HZo*&xcTy<>`@9aJ(9uK4kT7;Z@R8=KfI7QTi{oH2Og~>h ze7&7UF#NUJkFH}vq=BFKx^;f>Gb}e3Z~FEM>ecCqVuc<1sb$Ike!+!{VEZO%7S523 z?-lK&+4uihx+WfCIu!S>A0k=!#7kh@(L~A$6<<%)G(LzdreZIkK74p?14*B=FNDTv z2BVqR@^NL%?4|2Te(cx%o1sr)Tx6}y+3T7*)6}SpSX157qH*N=>b%J(`Gc4nVY$0Z zWF{05<)~uG5(Y`tR)>`3hs^a4a^>twsQGnN8je5s7A(}(H_cXDGrizfYXy5}$~5Y!xZ#ElBhY`Mv*xmO;0#nEcG!@r~>S=!NWyzCGA|3lp87w2imeYQEj zqqBTyGkr>?dWzm%ULVFwzD+dVHTxCtAUIytdBXtMN!%*58DNpAYg@%?9)awUNTjNRa;C5++29tO*E-dV_8<|Fj=oYmj4w-%Dc8E$t#{@F z|1{WXIgXa+VAO8EFT6G_jGlr(%QKy;_~Lk29lNNY4qgwRDu2_4jB%vHq8e_((#fh7 zE-BGyAZ1Mb1CCHQ8L6tbaB=q|o|n5&A+AE_$}F4v)Baol9FF8hmT;dcCWch~Q@4@S z(n)Y=8Dr@r=}ksKnYWHe=Uv|-B~Vh8`=ku-{JZP&9zVnx`TF{LnFg;`cQGegu7tBX zNtL@9_Y(@0%{m;TLF(Y(-Ts5qr}bjg^R?v=U-$-cTl4|+5_wBc?~CphFK&}2#Lop+ zmFrhC;u5({%%H!H|FpW{%)UqFtH(8)}QE{RnE>^c+-31QD#uB7KWPa|bl5McQ z?N9Zn{F24k+!-ynEyI6&%wQ0dKdd+*KvWu~Cyl*&v(&UN2I~8WuUOKnw%qtEEo+4^ zxi(EDcv`H&f;@WnSG$97eo(;KROM^fHz18ng9!zW~80)X}HJUR97a0Cy|I_J}yy^ z=wV|wPaBERM;_PJZY^Uf8n#vg@@g3p^S5EQ1^t?fY=e9a9bDx6L;0y;gLKon;!os@ z?ePfV1WGzDPn>(^*hp|C8t#^IK$H7&0$%B7QQkPNaC-30n!ee#MrAH@+)kRWz7yo6 zQ{Wz-KV82$K744UwLO88H}m)QXaq0ZS5M3wUd*?R!|`W?>-X8Ocg5}==*XY2P*rYU z8#yc=-c7SDuAhcX88vPn8_Vvm@22&dGnZZ7SDNBQD}3ygd8#`EK7(>geW;Dj^f||t z2*b|Uhy3F4=wW*jpf85C=dQ*drq(nC7GIc%cmsU~ss&#=^dFdb0wKtf$y5vyEmtCr z`yb{b&QaZ%SiW!tm!7|Tgl+TCiH><&4!#%R$*&p^_yq~Ekg@b!sB~8QQqitq;}3NS zSaqbA%IkIgd7IE=ORJppG~E$eIenGWekNF{B!fFZ`Yu$d(3(FDI%+3({n2WsKHd{4 zxlkDUvom)>yL+cP-g43AC;^hyF41hMRz1|Srd6cwTys66oLw}JI@3ZW;c;bzyEGFN zpi2AHu3k_7y9nhUWqy2=D=C(tL`2j&GX{-dCR{|e4`HoMJ=AQDn_~L z=F=^@CZsC#CXRp)>Wj%HsSJKdkwCj0u^wWfbUQU(_IwGOd<;Pt9WFke)>I?@gPbUIX*t?GEK3W+C%Y`R; zJ*5Ef_KA?bzLYHj3%PP8%Fi2qOEm(#KhlFdxlrjZ;6LHGssx zmp0DuRc=v=j;){z|HvaPp$ohUga;7t3;z=|`OYg~ITCQRYFG-5?{n)*Z|y-dCwvc#OJ3=KxoDiClVWnMTkj3KV}{zCS?6vhBjc#8KSrrU*zu1+GR zXa~|mWOe+17)NP}k;6O44ZcrLQz?wLPVS#NLO7ags^%|LCiZr#WN?q|U((uXD2ImLcF$koy&`Wsj_ZEjR(dq zh+ni%BB5oC^Us@fCANsI%f`z0(KeTmh&hzd0%#s|l66%`L;l2@0yBen6(1u{AhBfJ zCm7+#PjtOxeKTacC%S|yx>^9B$0Z_Xu7A*7CSJA+?Pq%Do!^&>O^=u7{nW!JXG6jV z9;hrRJfiyLzv_=V`(m$6mIIkz^QJTQbf_sN_fB5J!w*gUkRi1$JwM9Be!nvaC>fsG zM8f3YV+}47bgQoekDkYWd*e1Q{ZDj9w2W<4k-94o-i$XHn6YQTmFu%rTXM^tQ%(y_#?d^!xeb z#_l}c^2l&)w2O}dKd}v^D-PU{s853=BjGcUF}JlB0Kd8+1D*1 zK2M`Gaf{AnZRVv#b^mhm*$YDy}jF?7{Kb`D{~!IiUK9 zi2Dh4>rpJeU-TXWOTW^~mulbyeKX5#n2;M7O2i#*4dCW<7-SKM|DFl?{C9sl`72nSPUXvuN>x^YhIXj5LcZ9vR{H} zmI>?hqNk}hfR<+|#~yM%#P5CD>eBmkdfREC88jQ*>yp|0DN{HcI4H$4HWMS1k44zq z%mi~HaU_0YFT=HXS^Uz?vSR@p@NyesI_V*lHza~#HO*XSw|w*Vnn!-^T&H={3IDRP z!A>4YG+W3t)_*}M>fRdZt~4J-YA|9m)uoRNi$Ce%6RfX60eZ8&jV{avhLM^VuI4E4GSf40q~l z>Kficw`xkX{6?+IdFD^2Yy+>ucDEhG`_n*;MOzys9Ggt zoy70fzx+zJdB&0nz17%dvMdg>0am>Nh`mt0S_q_@JhXO?4$O-nZteeFu3=qz%{X7y zRn>k>64OnI9*_;*LLrzKoB1z z*{8UsZ^zli=6VZ;32PCeUFXrQ*`+;>%uF^ney@qnAyLyG~EDytVE}qiwE!9JLC=I^Do`T%GebYv9A*9 z1p43$jN_6y^U%Zf0F-M=T;P#IDL|VFT7~5BK8VjC4Ho$q9%Jq29n~0ClB#0^F_rrm zk$H@;Aez+R?{Q8+*z`%PJTdv?Qk!TZ)5~@p$3BB2*`*j{!gS;$*v+u~GWOG?0QThN<*w}S^eG0~xT-3yeo!fJsx{y0rz_om$sgejsM zEzfmZ2(F)Mv#((LdTVj&EEbVFS83I7Sd-k6v+VJEIsD_grNo;&dN;LS(}fBr~tJ+A1wS7ntan)UeI9HA`( zP}7+D3mv6n2G76tvXypaVSJ}gm!yD|+W!td4!|9MwFGba|BFQ<=bhl$~(i^-qA-n~jI5<$I|P#BAQOS4&l0{xjQf0b#deB~tw#g}^-n z`FqD91(}lg&XIPK>ky8G0W|Yg{!7uGU6{qUsyu{8gDQcp8V;|mYu+yu4-G^ga_MKY zi22KgT6L{B^W}^sR4Um*^Rr#+w*}dPN!E2jaeZ5XuQ4EOml(JnOdi(UhaiUn>Vch4 zlIvyMX;wOW)Y#YmI-U=T@+09&2se)F@t!iMmR{-!mHu3rJ!J5sWO;%KaHFeLx==DBvpJBpGqGhs zSo+Y3VfL|QX}7XvX@++O_FS0Rvhcr8xXuFufP<$F@#9`)kQefC$|}HpnPr8q3uk3M zcMC{<_`I>!lVJP`kh&>=enI)mgr%om?zm|I*I{( zWc{6@ki5b}|q8fr!dF;dz$qvZ(P((kyDOIQJoGv{xPu($;7|kRQlO;BX z67zP#P$S@5;iBmC?o!K6*i7U}TnwbLj(DCmpNJATl{!GogO%D!FJs-`C}M|ZUUz$? zN-$#7rzFZ;qi2X^Sr(Xujr|4Gk2hT$>#)l0U#-8^+}pB?slpZ(E+1n66{&yu!? zE7B6+u-V;!r{9L)v(zoDRcm^HV&TOMZq`^)`VF&V&I#{hs;nog13?p>;ofX{ArJ=fYaPHB780ltIn$MNQDe*jGlTnfH5 zkueOzXQ}#Ji3FIv)0UK6@^-&^oXgm>fa%?3{59yo#Xi zPwBRYtK?O3eO9}%Kh#=P#te$%j%RM4N*o_5sriLx5WK4YDbIJaC~-VgE@s$eG}5Z5 zm+|Vd9aGqCp_SQYaz0DO_mA^6(&xLTvyK*;*2xC-D!!PT62`wtl<=B1H71Pqt{1NV zJcFZh@74&a1R5iMReknrEaP0vyD#Qe@4Ng0zIdk{1qjT)HA?PEX_f`xN0PBTm6?Bs z{osb{fc)LrA7VAitLq~T;vAHBGHlf>4|lNWXv89$=rCk2J#qkKvx8m46tli3d|pYS zs=f15sZNy_hOA>kh`*t7V9gkLVE!jr^ z3I+T{ohqNm2n+pe)eLTh!n5^q#&M&3DX0;vcz5s-O2JEjPyL-*-3fW~O*eJ7alIAO z)CBahX+Rh!7JPZ28Y3PWBVImFZ08k=Y{|~%=`P@NJ6t1vfPh~|J%FoR^%7%W{*i+p z%2GD(VOSrvTy<44dSxlRGO@wg9YYK<^g^Ut7CmwFD<#~2dX=lO z{47ci?KY{3mmI~}{gXvf;nSMGTBsjz`nXQa&&bcrI8nOi0@wBIZozqGA;Qj1JJlqn z2#2kjw$xh9FJVKupDWZNZ8Vb)N$ozZ{?Lq$PhRatI#+}q6g6Q-EJaJqm@ewPK=7A9 zFAi1Bki(FE6u81)iENn{s6q$zJiJqz&Al8e^K^V0=3SmQ{Y1r`gfT(C%A^YKHO{jS z4cipeWuIe37tQC}JJJx{Hee^bKSSpF&q83W!AKO*fNVbYir@zObxjTSoWDMaHWu#> z?(jjn5RsG`x<^I*FR0oA1t5W(z6y%^Al0*FCd=#)8I1iS$u!!_5EnaGkeHjboGXYm zxou`7qwpbFEFPMH)>N8t=)qlD$PZi~iitqgLHVGU!GU*WbWKSzCRKtc382Om3{NtC zQlf~OrcZxHT-BxAVWBzfFFsk8g&*rRzNC-kWu206uOe#l3!&PDZK6PPq4rbCobuxN z`@O^GiRtN+3Nf{^pW1~v|g0p zjrX{X+rmfDt453ippGXldh8r{dAC6I2tbfF@!oR9_y2j-o+G z?-s3YMr|zoEy?i9jKbZQs*9Rv=lqN0X6&pu1d8KP3C#!ci&z3Vfm3?-^4Xw^&Y&}6 z?AaALfC$ZfW3szrYRE#9Joc8_C?~Pkm&n8>0~z-l^2?yGCfSGm3o!(nxU1$NXGC4{ zt7w)UKX;()=}c0p+ZWOaJa&{);QXz>H&vL zXgeRqXi7wT?Fc@G!fn3}&n*LCvCIwbNJRo_p*p}!&H!3Pn7N0`M&-Iayqnw5D53A) zZ$Ukw<6ncLNP1LI{beVaqVLLOvA%jJgshS;SDl*(WFVJB`Qb*nB$k|CIYQ{@ki{ca z_%u~z%tLTfcjBld21uImKu+x%d&w0P(8g7LUU;3Q|4O94CL#lL^CbiC?rY&82q5^S z-cUkA{aLe^K%x#sILR(3?M@)(-_sW?ibB(bzC|Z1D<@?R*Ih4Wl*0}y>^Y!73iRve zE2z$fNTR?9ql&6SPynPEZ$@F|u&m8L=>KYQQ^p1{C^wxBBX5H1BH~-5oyLVO$MWiw zi0Axm!jv?O#*-V;G=7@>#rzNcS-aUB{+nsHp|^cXBDTX4j18LxY2JL?xWaD zIENdUxV!tReKpdJgG!Er%ro>h!qHKKN=o3n!_!eqFA4E@wu2bddnpx`VduxuL9- zHADUR+tM#v)G%J6NDQpcJ^5EN15aGXqn9zi0`~dVOl6{vH5iFuFPY`i)Vo@{VugAv zbV{JOS|OnHo@joIIz~c@SRAHI^$J>x9Dx#9=UVhnYVl-EdBwP~LM$0{!Qm9}Er`S|dsY0Ts z8G38=QEXBp)&cypwG_?hQ#1zTA-LybcYB14Kw_I#=mXFE`DYu)*z1Wg$4kGn?2&#~ z2wwTOwCiB)`O}=q4;U}fJ=?`yR8ep=Py=ORef+u8Bcl7#&e9Z+DudY09LC3PScgFt z;)$&uH2Q?=D6S;nFO!7w(N^xL`=32xd%Xht(vq)kYYdvnPZBb}F{2Xm#u7mngRoy% zdIa{7zv<$FFB{kj)JS_l@-V{mr#coWiTo|a?z$)3Jben=_B`@JK0ZEU!FV+bLn0oc zA8=^qs$|N6T6zv3+2Hh*R#eCFGIm!7t8{&;dB$xj2V|o*3hCf^{*+# zG9UZ+Jf5aSX4XCJpEnZcf-3+CSmW312paqUO+3j5mrS=4lUS+8f)WEsry!jfY2S2T zS&dEd6w7Uh_d>U*pB05*k=WQ1@&R3`QDF0|ef81K3U)bNO7?74AwJH-Kwq;n>~MdJ+PigQ9C`vIFz--;d$$Q`{ovf7Gpqm-H~&p80TQk_0>_D+rt^EVQ#`Kx`0&HMRo+W}R8z;Dy(L>0@jHKp0wkHuFtN z`mDhhz~m9RONX{K=nsG@Kg&npFW-s1nfR$69^`Jxwr0=MT^)h{xG0qeCKV9L zfqis04<6r>`D1BYq5@x{CxGjz@Y)(Ih?!C{{Cl+!rrRFK9nAtx8Stjzl6mxX26=>) zs^dTKnSb7Kied`2$GudI7VrP4F~mzk-CRm%M{U6kUq=|S3fXHGuv_POzz$#Mup+m9 zr^~MG!e~Jg!$df55*3HuJ0Pr=w4?T3&WyaI{Ferp7bZwXGwbkwP#(0(iXpHneH;<9 z7$*7J%9ac3(Ft>|vOImQnz%Ca@^|9Vdru|lMqaXSQO{|gIATw@+}_*T=44G|TGLeR z6f1jII!l@?Gk5FSx0aXD`kWv1&h1}tVX=9AY-lEtv(!^3Ev5BnW9B=%*LJ_dwXw_y z$&{ENr+U;8bUI8E;OId)=j>nUgk0s>W1o6L{?2Qbd5vlID*yCdCtHhY^U6Q3T|aJ* z{&4r{4(ORR>22^P)y=tU_-V<9LocYY=wXfF)U>`Jt1yE?gJx>hV!7yE*;TN1fe;cq zXS9#gu|&O1<6=~*y2V~Z=xBH|`PJ^(AaD}89y}8V%x&|Cfw?We$bWNNRi09ps};zv z*S~u?0q(~@^uYq>0tiQ3n8{xM{|H5@5tcyG_jRQ7k8>EQugt&6@30;%t@+KB7w|ks z%Up->oh`yy#r|7L^fo8LOBRmvqB(oR)R5p_p@pXmdQIhJTn?amS2v>Kcb*>~5x3A_ zxJDP4k1x6;Z^@)X-DY5ZI%sj& zNBb8ABGQG4Wtk$&MtyJ9>en_K$5wSNk{qHlm7u?p49ASgS&k)H&NcMVHgXoajVfA3 zfZ`c0BKD5ywklZ!Bc_LjC^$)mAUNc-UF1CI@yG9d_mM*YbCyGL`1kZykBf*k&U?S@ z*HcMve1o=FSbNd!AL*luhEMb9NGs}TA(8kjvY1Oo{)7{2K}%m85RG|up6yRHa*$4D zXy2+OS5ne?<;pk!QMFP~xsDQrKa<_w*UTi3kFnNU-@vcFb?l!k;GP)yG^wCGw~!Fx4PRirlTB>2qm>=+ao(BDrBJ+mg$qV6oIC`=U!q_5vaC|dM6d9|)abFlg zlZmJd9Zr zO6#f`J?=8A9?|zn!Udc$Z6;D)XZ}QwWg-a1x#Re$s~=~OImF&zAknEcy=cC8Qr;ayM>g(w#h4(Q?8;%a@AwW5* z_;W4<(xx8P74;4)c|!_G&9JuZVmBA1(O!sa!k1=#u) zQ|`&+wZ;jKRpyaP(Z5|UQKEkx1&bpnIGPqX`dhdhW;6C~2uI;J%`Pz~tIgmwPGLDI z5JgQ_@T(X3R&ja{(=@Vwk)lao>7NgEjY4-=ch1PsRC}IqD@UxKrP8xLEQbdio^hoq zJ?q>v3#(J=C7Ui9_B`&5jrypzwNiBu!ZiR;`+RCvpR@vj>6Wv~0RJ9U#qien&pyFq zqxOtulBC@w0u=C305+qNdIKVEpMcLw5XF}AzglazpSJCDA%#?_kG>a!Pe3=S12r{- zGTsrR20W$Xj%k<0bB>NZ2V%>V@AE69z5I8X8ZRf1C?6P-)O#4yqD^ww)Wr^s3X&ky zYn^eJ7?I+W^cn&~S1E0a$7m6d<3c`%T6Dl%;#?1qNm1%1)}hlBnK8sJmM2 z;XS4WeNyjD)9X^+8uhT*1-b zVXqI&>V5JSyU202%-iB^r4Monj0-1UW5pq(bDo;69hQQKMlfTucnyLf7g*CCDr2=8 zP7iHA)V0%S>ULG6jJovA z6=s+EAR8n8Q*TkA!v0H!4r(33_gIcU_WoJT#N^1#;&_o<18w3%Wj`HyD$xtM-Z>*A z%zG~NLko@$^B?7@MI_QL^Vhy4F8Fek>`^a}H8;}OXC24f*n2u{OJI)GF=4`$zCATC zuBRMO%8QpC4422VncYA4O={)5KQy0pq-PnY4C5ZB!6Q8eVCBj+Lv-5|{7uM5CQ>cO z%*SpDsdX~a|oIWn{S%B-+sZ7GLEb(sEn&75+rNq=L6hn8TCf8m}Ladz*4SG zvsSwZ=}xe(dD7eC&!YK1rLUqxm+|J&v5r$Pz9L%GV!TgCtQ|+F3lEiydM;Ui^B4>- z*5?F$75qt5Uo>{eo)v`kQMk$>CsIyCZajGYw2t``cW5ZbG7Zxo?~J<+$tHX35BNh~ zbMB3|X#aD7Q#em6W4w*vSDuNCOjbj%Utg7qak6)AL4!_Av^Ph_$7KpHzD;xwjZb&Y zXXcW(HgI3`PJY}T->->rncfpZuQS?sM6aQ$P`T0HjwNQuxuWe;Z7`q#m6P$VgFHs= zsFk>C`vAjxu}-kSqV@PJ|J4WwtVGxqauoSPk}wojJAj^b@rB-D0#ltU7(J%W!xWCX z+RFuwnr0nS{Q~z4kiBLF>}iRKWQ|HQ5Pq!QdXDq?ihemR-CA5P-FkF#R=gH|Vk&o; z;bGWfGLFV#97$y{j=pUH-ZGWlxWg+o%>Zd<%Jg^jDWuro6^|37ZNwL9!>!jk+f%bc zr%O0hw>iddCE7_89i?c~a2i$Ix?^2;XKQ3bLa`Tf;WT`d>0>mw+MlS}AVJuzNU;8x zgl5;su$4jtT1c(V9p8=1lKgHm$SW&rvDv<}r8AvZC!@-W@u#77o?o_)b1JY)obUpw z5|UVxD`IOVrxUUEdvfB=O2VF)Yp#Dt02%qObC3m=7fPd0Q!=7uD{^U2e_;v#^i7Yo zC;9t|{w@aNDzz;7E3>*d+6VqO1N;Z4XfToF^H7_A5Cy(`e7%x;y+vw3gq$;a)pLRSn8`KcZa5y$hY-rvhKoQwEv9NBC!4CT|wkgOM% z4AB8nQP5l}qX;vv85?_jlz<)nTQp@RzlMl;#ijRkz74~Sj06u7G-C>Mvo7P@{R#CK zJ3b7>3N-ZccNw1Zs~QEraYHauDf-B}r%R2dv2A6-ahvZ^6Pw5T7UM)=a;Gw>m~?6@ zjlsyaqhU=4gX%W=WzBuR|1@Fr%NwZ{))UOEN9i~YlaCr6{|yiZUNn7it^BmH<^pQT zYbwabJ6x`$Zft5aaI>5+YNs~7-z95l68@alpF8_9!|G9!we5tHaW&9pun~8KQ8ug( zbsET$wrwd>a?=e?uv}p`bGKi041j50ygoi6{fbZvHo;0J`~?dDX~+!o?v$kN*)+59 z)Tw3sBut&8UUpp>onk<3Am#oe1|zgw{f--Q!y48HCp}oSX_1%2-dbiQsfrVxImL36^*(|}B|$$W#xXGpmWUFg#F?_FB! zjtyR~<_KtWxl7ODif?*E{%uel>i5mR*Y1OPGCF2#j0}`XkH(^N$r(pnn=@#;Lb4o7 zIZm17q%dOrYA}$`cnU!<#iq+vOSt-E7P0w%iI8bI4u&B|W>J9}GzorpEuUuM@6tRa z(l@Xhh^svl&!)Wz%ANXg6Woy>XP;$GCBw#+G1$%P)=D-#6vC6L*^T5H-qxF}&PUHqF*o+L;9NQrR@My%vkr>g>-z*RlnRFeJZgxoH}O6>_C`mM z0K>Xk`{{DNrAG;o^A~JHts>+coM&-=Nn7)Ic)i)G| zT#PAxv`I{eoHLQlZ}1 zWO-g7?6dPUNs!UIcVchBb4*WG6MM(DV6+g*Zpl+tAWZ_}Xf+E@n(Tkw!oRE8hNhaZ zzuUm2J;3_Wd55jYN=0@~Fr>>~z%Aua+;}FsQGTe~W#r$~3P%1$Z;NVylT+t03^b*c zy#2#}CiA%@ph}jX7mt-S=*BCkSDL|BU+jLeeKPJl{!vNXLgL8Bq+&xL+VCG4JeQmC zM5yA~#P~r!8LN;16w{^q{L(D0NmtTajSauAn(oi}&lj5G9+Mm|z3-31h6b9M?$3b) zn(p^~8kuZQyXO>|Z`STh4CM6 z!m9=OTnJAdE!ieY;~>I*eYX+_oh8jxBx{2scApmR4DYA4$IHY{piZ3?h6KUF5@!dw z=2T)$cPbVSh+PAi3 zirwpPLm}klPCgvgNzEkHiCatM(8$m+|Jt(S*9L!MG6ddDFLy6yulu&4+SWbE4A`H0 zH3ozn)g_>&kYzT298=g_4`AB4lF|=6f%&I@6!p9Lird7c2IG4)#@^LZJlYuJw;_?@ zPhKN9I`ST2qYeDKmKM;#5L9ixkVr=_N8|cmL`Hw6tb1M|`04l3JNi7>qq*HK?(mT%_mbw z^!Q~qnzox;fo3@C`#ob4hJHhmt43und7F@6pbw<-LrWi7yj! zBstRc;j76DAu55}hdZ$QRWk(SY8T}|dKe$+g*7-DD&Ew25hyg+9T!5OE-4V zX?3-B^Ib30qF=3rxYYpAloQP&39n_Uo`wK5ElOm!QK&_fQW{fKVR3lU$$N%p18B^n zJ>bjK;gc1nx&^BcdIUFR-;YA&ny5=Z4h~cCWfe5zY|FD_nOvyRIK)-ETLOT=%IlC0xkO>VazG@!nStGtr9SO_b;`E zrzltQafZ*@Sm7kPU$#cN*($!7KWW=+eL!oUVue?E;~cM(B~q{!kJYjZHnJMD52y|2 z^Ig=77;a3TkbQ}?B^@;4bIjJVKH)+Zw5H|$>Hk)}KW?t+anJzK#{h8A{;801o%blK zImeMOF$1&FhA67G?y-dy|6Y}lvr)2KtzJ|y^>Pwa&ELRCW}M6F;NsKF)u;}+Q}dxU zRbT*%XIe^K2>tv$(jwPf%)@Z^)RvjSiRv*O&P|;22~QPR95qJbqccXb zm#3QTaNFq&WTtev5Gd*VS2m{W;r9T2Eg{l)VYeqmJ2M|U56)-GaWxOlDG6i1aMLM1 z(!@d0#$?Bx4loq{`+{{Ymr*Yz5r5%k1)-?Mao;K5y%N#~-e_lW;dag@l##9i{20d@*<|U$SMnCz51P>_k3Yxp!#__pnSR@gL7x`=MyZ{;*n-7 zFhx8!2u!_IVqH_Wi;R;ZWc1cWhjH{q z^9bml2*a9+=t%{3bkh+^o$Ry-Ar@PF(cSH#2^IPtG7G5>q*#r3kB}^na_7F72%6J| zC({eL*6O4%M@bkxMp>ng%Ca%Od=et3#p=cA?cdA3O}Yo;Uytft#$1!{J}7>-xz%J2 zuPwTi-pDUVIx)1jXGSTnduJKTs=0k$U?D`l05B18-GLC%mz@u+uC_FQkdZAcav89j zrWH4e*Dgb-txiFSI44ZSB@yG*-R&?&{n6P5A%oD{d^J{uB2>=%wVpSw{wB!hlV7!t zb!2W>ig@TzH?r)gjDW+92kNXE!Bw5;(Lr>d%g$}{C8I~C!#wG4r<6+Zw2r=3N{(+! ztiY7$fPGNOKP;jRq~xFxUyY}@^ccv&+O%wM)O)G4-FcWuBga0^6el@|f~COD5+9XV z@)fS6(-W$U*EZf`UlmHyGcvYyh8QzkecGhzZ#S|+CGl)^YWSBHK3wmKh;rGie`4lWWSM0uLwl!j zM6}*U-Lm}aJ=PzS0RPH+q;)^x25eYSNfbTapNG;9C-%vG_7NoE-bHddcJb3MZkX^I zhdDcx@Ec3uZ{j#lunv00dvD-(-Wd4_7k8D9uOaJMkD1>_fEfqsAMp0s$*_O2ptzED zqG^8?oM}*X)f)%s6IX@~VsX+69@bPUT~)FWJHHSm%!tjIubC65!E8rsYwD zdm;;g<2%o@YTQniHbox+x!xq;g5%@_?vF4 z?>0UP$!goEVzj|P2-S_8#6eiyvc)xT%6qfwGR$MI+?Qyx@RuM^uTV=#^7Yl-^&T$U zjl|*(wuu&*U7E+^Mo|~GZ42`ogele6@A}iEM(H(LG!ZKPmq|Dfx2ic;!=a`7kpC$| zX}|#Kb8C3OZ9Q)C))j2U52bt2Cur*^ga^lTYqGa<+e)&3wj_t^Z6gJ2#9e{1aTK9b zm9m9u?W+L=X!SAK2PYVovlT_Il{et$fwF!8WLd7+XrJ&T;xo;U`X=8Do*G$4PY5Z~ z=t~}?q^uHd(%AF=UBQyQcPn~{D=j(vaUyBWg&G$hC4DE!!4g-1%p4#8@?S>I_^p_` zQ?B&g+<(*cMUvAMbi25H!<*t`VY>_7_75+G;Ltu$^O=jDpPJ(=Dt4mkM0oV*e}?8? z4Z(iR{C^#+TM+Eo_^G*K9&1<#ePbpift)EG_^~?nEv0F1e~E6NXZ~^G+vu@J`p$^S z6Z3~>;?ew`k zpyLLbbkQlAM9S7_tfN5s|0d9NUzL2u215y&31ox1Z7dA1TpE~%v%7_tz>M)Eo zF*vxX;mM!zueZ(kVqipATBLehQ5B8R{3>e!?9tSO#bPZYXhQe%)iS!PtH_*xwl`OW z5>=zGQxl!F;b~f#2q+?UCz5uZGKJsiQkd|gOK#NZ4T83s1;M+AppQ)x!q?URfrSKp zRyIV2Gxle9R#QSzn^MjZD~bY`xH-VCricWsw9%rNuUbr@k6q+8wo513;Nl~D+>hCe<2U@$_*<;)h+?7HHD5SNQ*8n!%i}3JtF?A z%6j6Y_K;&nf{k_yQ56RFtL-G^aY8XjzUmv(Z-P~sf{~Za;y*3LkL~18niQpF!&jvo z^=_w)7S*48Q}0-7(2*Wn$$?r25pvxG!?eQ9(Q#F989j9MP5%XnHFHhvnvg_{x4vb2 zT5krRzen;(L<7$wcm4z_fFv$T)vp#o4E>>zQRinKULB^qz!zZph+-mxS>~7t#^>k) z5O&ZQr@sq2rJ~}SqKin5#3TT`cV-5m9lbDg5ku}3$=K;Rt3)joGC z)8Q9SPiH(J+fNr@&z}i1?y(wGPY!U~>SDN5#QMVZ(Wm%H*X9ITS&8)$3zHv|RWvFK1<%mGhmpDG9~di=r{aHW==w zwdAK9Mi6dqYdHAtPl{6~qVxjusyxL82m5{L+w;eRh?izbf@ReRjhBh&&GSNgtv8Rm z2-s+2QPk|}(()M^@lydld}q|SXT|DL7(AE8Fg|lVag@Q@zvW#_Qr60i$MF*}k$jP= z$ehBRkVwn`kNZDw8$>Xgl6#1cy-)5;Zr!auYj-BZPSIVh)VV{&UgaF(yk`9SCcOw3 zWRp)iK<$=MoMzj%(nic(@CX{k`Rj=!X0M12A z`yn3buI8vRcD=u}xNDL$0&~_?mZkZ(W&LSW=L3?TO%2#u{L5+R3$rkC-#i(#B-HfA zlf)GO8Q4hjIRXc2N-YdLO{~)2;cAM2l}3upNW(1Dns#n$XfRy;I1bFHkKcz zveG(qE~4q{cg1a5qTO0hUTeQTkr5qYDHE0*mx)OtI8XS~;`O&X@Qp$v7D>t5m|h8i zvHHkV?FQQD^VL${8gS3-8rpThmYAip$k!%g_tA{8kemEpTBPrmJ*v}9cH8UVMd3YS zf*f|eHAX|p1g6+qkG>AtGaWHq{6O5ozXPpz4gMhyC)&PD%V^*_Lbc)UeX4vlMC} zWUsl@op99K@wFLi{~2g-Y`fZ0qn#FTvqtE4G+ZFc9+H zo}aWOlg*YKEyg?+BE^$wB2J$VN?x*t73WB4Jj|lVv||S7NZyGslrnO9H&%aQIcHG$ zN%R@V+@jUuWsx{%7cGk5ma>41LR5_t`s2`mug@o2{g!E?c*9_`F92yEGj6uCHn)3) z0}iYs`xT4q--of0EOfjOoKs_4?ehIIcZtC~_anqs%rE_WSR#?CzWYnZXMnx)VvtFT zJWe}^X}a-scAQ&)5J+!cAaSOp@Cc$^$6l=RW$kY;-1X@3ZlN9jJM|kPmS$~0!o2J@ zYhqERzw9f}>qeiWMDOz~Z0Huai|`Ci$m{W9Y(%TOPRnicfZhf1wiD&{#@42^z~WZt zv*nldwt;iY%1w2(dcdcF&S(e5PT`_5_ZsoSde8!{~*V9qTagPt(57);f)ay+GO%MCW zIZe*b-WZyW*If#UOgHPt1ey=)$K~b2FA4_4zA?}e@c)N-A<7cmemU|j)u*DwgJ*;o zr{5}3q3jPr6in|_%@pf+F>`S=MQ>G`5QmA%sAS3wF-c_nU(Cz;|6*P!y~kwzY(zK6 zdbrFbE|IQ!QU3(lRkm2a~RWKb3W+du)D=h?-(^IVpgr)3sPEZN`Gi)xJn# z_m*N8#}(uB=3K9kGVV7I;r?v&Bx5;O`;Hgp!SK(Am~c`kSAGsBM=f<%AH|(vu(fg^vTffPgBEvTUZhK`6=^;_?r0%gRK$mSL0`h$HxBA z$FcSr=lcDLcR3s`1Ltq*Z!e`WY}K?`4Plzix{ICP){#^SEW_40|T*m zhK!b3poz7q;Sb7P0Hze8yfv47lAs{PFv)lNnmeXDo0;Kwv$N<0fM*~$dOBb49xjG! zS~4++3`)y_rgn}oc6iB4;sslR%7=$A&(#!Amb~goI;z_?9wg*BA`yiZ)V*}wHOL8b z0`OoRTgiM4Lu1qZP}IuXzSF@$pc-`_b%?GIu?+0Ln@s^7^jD2tIP2V&9>W4Tc>ByB zGoy-t+S_x-#cBd;ltBv29U%Qfry+-eIHktY4NKywiB$`(~C;eO~F#T-R-;-tQ@+2K|Ge0{Bdd(D!**n@nc)% z(n@M|BlB0txGm;nv<4KV`J~#|SfcLoakayD+hPUBzFCG&{51y3u>u<3Y%{hwB+M3I zC#%A*+hwaYGElDzqug%h^d*4qOwaTs70Q_02B7DJw!5PQ06{k5tJIxaEpTenohz+AfP&;U3CcJGrfmlcth>tC;f2zgs z;2&z;3pqF*LNe?)FF5E=_xlOKvobCjX}!@fgbA{e>)7wf~D*hTZRBq=Wn zobd1@;M$7+0A|{8Q6WuxHj$1Gm_Rm25-)o*D$!?2VdNNcwq-0t{<~qO&tcG=PhoNr zVPwx9m}(ywTTShaa7*f$La%qo=C|)55`*$RB+e8gqP=_&gb+j$3{d!|a<&uZY_`&wJL>AOB)GRT18g@=@ zV9h=MnbI6VwpC**!m9a4F*<^zn=!`*2Gg!ORX;$WHYDIfujIbp0`p^tU)w5(ty zzVV;5sNW+RwA}6Wb_&Msa~SOYQsPT_fdb5XZ*@BeKGCXj_b@Ue@CQdNdj#6$xfWvB zOChVQYmhjsq}cbHm$i_q0)yv;*DDdLWW`J^IcfJ^#S@_e+HN7|vMN(}BCqZlR?PsA zA@fBgF|HKkXC5Xc9vn~`q~ZAHfGvQ3szl?p(N%yoE7pQ9ffw)>N>ZDP&|TrR<^*1*!RpH((N0qPp%9`p-fkcJ#mi?D7)~kZ_E_}zmDn8TgJGw4U zW8WgfdY=XI+S8RlWz_ZhY@=pC)0glqv~}PSfUjV1)%rj5@jR=rhawmkoLMdF zh&anpGf+nxQxAEs$M{Hed4=&9Z4-0By@1%5xq-1pjX%(xa^=AO{`YonSpds;&RYN* zD2zq=={iZeBSEfa(D&C+@}r}Up}ESWTJ^CxiJ6r`{mw27)4G?@q<{+87|K6eGCMOX z$k_r$;`_}+jsa8QgOONC%_!lK+t_&BWGhM66vQHD$* zn(D$DiKPR3ei$Ybbr2K=<8;0bqhnK4NV(Xz$oJ<|OjiHkWiO6Nue?{Sna@zl%Chzv z9=EcTzI*070l6f-Mdl*5yiCQ}LE7WSR9pcX%eiW0AfPs3TQi6p2#!YBTnk@QwPebf zq@a@t{I~*oa2wWwnY&MkYFeBJsnBh}A^UEqxErhg5O z)~a*iW5c!NrSVRhXZuX!3;X8rAj`XR!KKmXh2!gV<#(*>qu66hqS$2*eTA_`oB!{~ z-*dg`o$}rc)gUZn-3rPyM`Zy_cC8qoW&SO5>82abQu})A~q_ zx?2!xV@!NNyrY>M0_#IDX8fUOBg;;BcofBc(d`>SPrqivHZUDVPj5v}CwexMtPRk* ztqwa~g)J+yL!P?*7U^}Gjc#&_L11$4SeF9H>sxxJEs&_HNH0v8qH`k(X+scLu}hCf zmQT#68lB5rNA{RPXj8kL^0Ew+Xs8ItRCc;i59n(-(DJ|U)SMcLcJb4&^sx%*_0^)@ z`JoP7#qB1}Z{JEXquU=(@P{s#2Ts2T01RgXmaD^`ke<{Du_RpClnY=p(p(6oCxd1p zz1-Iq;2^Y*R`%>b;&v%^jxIhM6KdN6J%5=q%MaPYwadIRAC+RkAA00X7=V{RTOV5w z3Rdrwbu2&UBa}1Izu*I=QPK3c4J5|Rit*~aG4>tkT*sHU_3~t^ zf&AAphCjek-n3N<5~f8e4@($ZQn!06xRCMYVmFrwEB%>MyPajB01PotBmss{@nx;FKSdK6E z4%pl}KUJ6DewRSJapVw5K}I5cRf`_;X7CQRuKr}-Y1SSJ`xcGy#o$CDPZW>(2|e*H zx2xvtRKZB*u&!^y(M(pOam8LQwI0hIe9Aeh(?w`3fV+bY^Y(pDVBO;b5Zb?`N)}Y2 z^#;ucq|cS2YCtPeg=0i0Tv;NiWbT^kg2mv^sG6Db+Dhw1cCb~>BIEOb(^AHjbH~*d z712Tn%odm9fEh1+mDV@h4%Ei6JhC*%j%>@VbO+{Z-0JB~OwWVJStju#cblkHg_8Yc zGv{CNwZvzw-u!b0X7s4@@_y6zMAd8jT0_mG@qv37=uR15UAjyPu<1yZkbJX%6}EV*UxwlSE2c<8&rp^bjowaamTGiv0hNE_ z;p2Uc;)JZ=*KUtK-#2z=;pV7_I9YN94Cao)K^v!${>D`Mj2t>Vaa?%J`0}kpgRAUW2$YaT!lN_k zmcSIN)2p@?aeI-o5*5i)ZKoGnrItq#m;d}E%Mlvs3N~@5zDCE9T|&C21Z-o@yXRo^YgLK=p5nI*8zUYY$m< zmA)7mN#;`&6y4orzJ@VL6Vp1$JB@@S4voF0(LZ-n8*I6sm2fNsv1+dCwix;9tTfSZ z$9&jIXO&t^bf6S^jpKZ#z)~iPR?ulyLZZ2%ZaN~A*SbZdRtf`W+T$^&zv1apBtQmE zEtD6M{g&;g8tQ$ptR(Z2-CQ;pwB(ZRT&Av5zhQ#e@7B}@+SI#i_T5qde(Go${>R7h z4()-ZDA&eY@r--2_#uOGXea8`%NC$Za&@fkz60U=d?>x%LqXObW!Tdk5WyG=X?olH zu_aO@VERWY=E8I`66xVEBo4!*w!p-NWTyAY9rN4QaP>|jJb!phS@68?6?STV9iE~za*j!6zU|%VMv7SQx*fAjG~PREAOsm zvHLf^E3nbSZkeK)d~&SI5=ex9G5r+A&O^P2g?1PTEb^PH(KH*QU?Jxpg||2f^Nk%b zTAACS0j(S{0yj?S(X~JeCUhM!92@71X3YyGa+ict`g)=!aJiQabTdT`$WdLaRvnZB zd4&V3SEEWP6jUrjNURA&8$Qla!q>p#x2TE_S{O*7L-$-WU!(l$FzrfZ7Q;Rthpf^ zx+;W;qj1zT>B9el%zcdQf7)-9L9zT_2wGxi{V)C~N`Ay<$^CycW& z!cy!p>U+6Q?7f1jAteHh@41q)>qwUVleSsmYHTt8gf0y`z}qNIW+)|&4lz}tGLe<~ zcQe^;&L&{7?;6THU;w%_cqdC$aTLj9H=NFDAeG6BX|W#(aqSMqMBIvrv>Ej)Zlvx~ zxBm+J>%YS7YwE6b#jiFD?DAz*8TPP;>WE&!=IOZf?|%;d$^1IVts|Zu6&m@U!#ZN= zrT=4D5l-RzH#TQP+YbFZJN`FmzW`ygDck zJ9s0Q4wan&nfZ9kF0RUbYfMgYXeYHyf^6V`o?(_SpVk%$F##^}KWW=*WRaL`?eu4- zpUs;`k2hiLOTMl0AK8Zd+SK@H*?`^Gts7b6+8$jFp}ebdabL@D;drcp@I3KB5w!G% zSJRcY$xV#J_uqr=<2VheNTJO$)}B>T(iPDOgh@F)z1(Bs^@giTHk5F0h42b;I1Aj_ zGX(6(_OEi3550uv@^G8uLyCwNI7vgn^$^k>%I>><4K^k;*0AbQRKW9X({FP81n8ch z)6g?DbOeXv zaM#EZG$UOu$#?5-XWe!CN;+O#Lk}m?&Jf@}Q3tCKd37|71+p^bE`b=rXh-%GT8u|V zEaD_<^p2cTO%^}8KRYwi=B6w9WIPJ1{5nUz^geH#t6A>h4wCC>5^%NnUM+&jGO=CM z^c8wpW^fj#q6OZ^Cjvb3x93|jc5fA0uu-{e!n39Wppuh#O99U}ubyn%tsvvCsT|6m zA?^SjNcH-Neq==kqL*g44W-8E95f z)|VRlI^#lfNu(lv=ah~tjW5BKkmd>cd4zsDo_b)BnR82Dmp2O{e86u|3M~ddc@)8a z)OrBkl1t9c^)J@3q0t}gyxrVCOd)N$KVikm;4Uad~24PEuM7v}>p?SJP8&8V)qeQ|fbP6-k{)p8% z%s9|cNtp&}Z~vPLXV1(P&;X^>$QabV+F|ABVg5c}#OEm9b(a})^6BC=JQEchh5To3;Lgjm$A#h|ijZ|CnAeeWm44-x9xq z&wld>TxG_gMKU#@{|hAnvaD@~eRMmhZV(~xech?O6}F*B2(Ix^K~+btGx={Fm!rs> zc>*zmnD}YI7yICwT^U`diqTS_LA-hn-*m9nlr4l}^arUU7Mtn|3Q@myMBRZd*F?Z~ zL7<2v5E(~QTf}K6W2o&j9rXkIxpQy|HNA;h8Ld_eK=IWWBl0s^MUBD4OQ_7t?N6uI zq=WQlgY`S#4yR_?3(uzSYlA7oWU-S48m~2Nr&6$^9+q_|%IwN}LFsfe+1vex#fT4D ztz`Q&5cAJ2ZZ~c^%7s0~L&RIb(;#csN>0gT2SB&oxcJr3Qk|uCSpOy!qbHnL2FO?Q z5WhjJ-vaIlP9&*UHfZhTJCy9hq(k`Mz$@z-%R3q=f@zcX!Vw#eLRjsQRDyv(&pNj# zJHT))a&?ak5?wM`K}ebD~A`zmife;7f+V7*fzWr>u8t z?SZg8{_)K2a<5j|y4`M5gS4=l0MS`UEL3on5 zCVeR)Z$3Q?4JKZ4%0}~O`tvG+?uTm@r&@jTJdX2QH313TmPU*8i2^CJqi&@bW(-E* zbxT13`E@s@i3jV|BLk|s!!WuSLkqOYnZZ%EgkFf|u|-T>qg6ZK))+$#34q(F!;(Za z76jEyxR4P#cRtY{N=Q71=fFHXJL$sMUazoDy|n*cI{Pmki&$NM!nzt4lldCqUImKr z8Suse9a+oTS;_JzXWx-RqSUP*wOF##MJiv4SYRHT;X{*G7{?>cID<8*bd$f~L$oR| zvn>v~LQhg>23IaxjZO8p-P`g0lT<9`V?3;F7`pXrPl9`!U@GP#;?)5n-Da+j&Z9#Z zsrBs61~xUq42-*&jYF8;Z!^8QranB&Y0>M)0b*cP0qdZ|MOF<~lKu77FXbR(Y2uOh zkgkQBbB3QVj!x}UOlugmVHTF~GeFD+OzT#|E$_Z=!Pju&wBMo?xz<+>JCba#R@hua z3)+lMXLsA{@%~WKP8-JAI&jFXZ{9!Hy$!!TMNKX=L5m0ajYR3o>nMy$orTpJRG&`1 z_)sRPb`qXY5^Pf5cP{6rW*s{Hk?sOoh*%i1Y%F{tUOM%!N{Hk?V}9bLC^P{1(yyE> z2=cvXaIoyx(ND~WyIusZ=Dm9i*gN!tl@=^G>d@Y%3M=T%N;~7XG+!T~ocDKsJPh<) zgC1_ij>9Jhdh?HS`;@KYj&$evX8!fj@*Aq6^E!g_mZbGCUXJ+rfbH`S(3dng@CA4y zi`9@f@ih2!z&HU3#$UOhHg5DX-E$zVQ=Mv(zPLgA&o<&|hdMua5Uo>T<6(!u;s8H+ zizYj)at*xA>J5d84@L$3PLJh@RIe_4$cv9*MV!EdW1ionJAdm5o%Gi299X!YgAcTy z*O`F<9czPU*C%dUHv=VWcmTAK#Jfb(XLtc5?OOLmxAy6b){`5+&|PQz{QHZ=H~I{}u+- z#T-8LE@p_~?2#(gext=3+`qbf;RW!)=a*l}=ofr>7+iV>{k& z4|6jLuX{2hIygV)-*+bm2L%N=-9ArOQ%4axr*}0xb^1;w-%l5FGKWJnae~n?J>Le3 zFMl)R=$ZEP^^3pBFv*Wq#T!Ic&jkeIE@ndr!k{-Tby%FCv>QN= zR_B-R@(gYs(sBHbBRHQJ0F#H%9d$aQh68jzvM}NPjt-92MI{G>coTCrL;M@|g`53I z0>{ansogvSb1d)PCp~eC(_1>AI(uPl$qG^`XFFLa(*+ZLuLjhvN$!ElGQr57q?N?I z{72%Cj)kRnK%n`N>G#rHDu%U&8>gsBN(uyGCm`&sM#5-hbe5o9UbL+ooMxjWPA}&m z-?k@kgVKFLq8>rd&yzmu#FE0`i`T7dA%I!XNPBJJ)7^ZZ0ka-+)V;JZ%T8sktr?qr zbc75x-+bq01a`IKWe3T776wkvq!Hg!Kg3M*o$X0z2ly@!c)qeh5W2}~$HBWe9I*!Q z{02`_ffR*Qafi>O{&_|IZqIkT7{J7@fvcvxhGZ(9I`3lWYjxs(7+6&Ote}f(Ies{- zC{tmZ_EmrviOQYHrs=W;B0Mc?(ZevU%Q7gd^&Y_X=rPL=K=n67*=sp4wLclBQ>&sk zP$74|hr*^j6_=bs`es+#lIFCo@nq4WwT0En2*JYQAHaMMa+#urV6zM)Uoxm7b35E%dMpCD#pAUnA;7MZfk;>R2~M`vi7}8xWf5In&lQ=bK%vUuC8a=eDKQ2Mub9+P55Qqsz#WBCy?ZO zNd~v}0#X9-4Iq~6gG9z=!ocYU>1!7n2l}Ol1x9PZ6g+6W(oKDmD&o$+j*2o zKm~<>Rrr*+v&%IaB1v2jQ6%Rkq5U5?5^t5V`@Vy&u5v=Pn!Vo+3O7lRei?Dz&7MeJ zgJh^j%DE4*#6XGs-0z|Zs~GXor=bM7_d0M%twT}R;G^d@#!Qa6QfQXr`7n$I52Zrj zqev?-jf_UM*uIQ`G@@XN6nU%5VA4m3H7lPWbGxv_b(ylt(2iFQ>XeBHV^`)u*HDNulyr;o;z1&2d?Cv^YTm-7xyDQ@?P|};a>LwZp&-QU~ zaCoDVqiW74V@m|kvgM>z#25FhuMPD1&7}vb%i;A;edytlUH=5&j37J z?UZRFeS;Cwfkwz8vM}#8xq1EGVf<)B*s+){=cOALjjOP|d8|qcG_Qd>|H91q;@;_l z=(MIYxq6v3y@|aSkER4yWX=G7vl~gwTJVEUMs7b%SqT&<;ZKz2OF@`hXB8v8oX`0v zESm|`*;qFPAQ3#Fxnr>+2}ubP9E(bKfNc zYpa_%Q~cf{dlRPfoO5M^imtgX!z3lxUps8Y#+n@6(KfnvnN5fNW7cK4z1Sw-g;2%l zb|0F=`;$PLB`%0F=W779^(y(p;%|GPDD=r{R6=-vabx0SJRx zz#q!^!=g0WMC{6K?5}Q@*WtEG_ZsiGmC*33y{&HD^c~eiZ;0!eIN$3YwQBeG8z#K$WKY=;bz4l_b4Nz_(TK6v! z+1S|13k7WN{JVjff zB-_&iW4?9)u6|FHw#ioIS-ZM%n=4{}iXj|J!ltw7|Uicg*PCOS$!Nlx`7DPTLk;wpv50Bxq|F2Hyi;I}z>-MwcM* zo?vsuTXy#~D^E)9nVf{7-v;FNr#`7IrL&JWKXr@_Yt&w%P<|N{2WdNF)qH%2zf_+w z&BlG88NPXuybG7n%(`nbjtvg2M;&+QCqkHlOdGtW2yqJk8r0jQ9}F=3WnO8Kx}!6X zpHalI365z*g^hK$Gma!}w$wUgk>ERwH`!)bX4R=7EvV2hYN=Qh39FXy>7aBH)fM)Z zLcdH+2bY29Y1D2uYx+-Fkcr-%=PFReMc&cVyik(t0wZrjRvRW5g)dHrnItqfjF z3mQFeJ8e-(nK9=?W*Ffo(|CY3{*w{XNV^Rt5GD7J?o3t1%S6Qpb=ou*c9~ePOMD}r zeI737%X^8R$A`wp5z2>TZ);Uy-Wp*mwc!ZDf?wV57F_YylD1$6K*6nZfi2;5^McOt{tN2wlXx;Tc3*hn`%l^Q#oG{AJnnV2Fl+)_jk~mSz(Kh|dJTbd# z*q8j=8dR0W3gU-Kaqe~1=zDQg3WS5~@R_w#tPZnhx{C^2O z0#CyESw! z8^D2O|0rO9x!y0yb;2>`FxIgAXgAJ$fZKK`u9kv=oNP_)hM-6!0jzAXzg!ap#@tc8 zEa}n9<*K?*sUZ4PXg^bKoDn@DLa>Qis2x!AZm~n%Y^JUfxIe0DT%USGn#T`wZhfVg zCaqW}Brhc+wfs^ZML$$i3rBoeuND9yqu0;(t`L;&`(XT~;`GlIjW}P~aYNQ~@@@^E z^h9~0FSzIT`}@_z!;kx2z0X8f?^t~{PBI=eCy471Vhu!psqSMB)CC*|xtaT>%p99@ zKsp+79+u)O$abwN5H*Bh_0D{)Ymc;Ev8}U{qY;xx3*-^t$CvF-HSNJn6kc_DJYY)7X332);?$}Nkhp{)iW(}J z5h6){IU?r0h7>q}a3gz-%?Vo+0V(QpA=DA-a7jnV)$&oca|5B|4zZP_VxTEFg?{s@ zm9H}|3W}DZF05Tloq8{LXyZdw&+GHz>v5Hu?uf+MaQ{IsTZ#cy zuYTE7*X}gb*`qyttJ!ER2MEV4j-9HXM`e~M5Ntb*%!gB*^#GGTi~2y>Q(MK>9y~&b z`77NUH`_+g^7gI`^O*xqAiLSy_o2wR<}QYzRLKN5w-*P@%m>?@7+f=3v{m(57`&>%UTbd8|Z1;LF|Ij zIqg-c{+3p3SY_s~h1w2IMoU94fwL7h@IMAin$HZYBLMenF337_g)vg83OFIf9>Xeb zd7=SoscH1)z)5sqw|lNFfemb0s$9>DiisJC25F1sbzH!g$%d)#UY1fVKMbzefP&;B zJiZa*)mf}1Ejk3WB}V7k41qGxo?;dQxuDRgc7~d-MTmp&)AvOxF#Z6XJG7noVH(#D zlm}w1q`C?S_B%{F^H{1?kU=IGY57pR_yp5{tkoC*Kjloy8%$o`( z3z(S<{@@Dw1u94JB`fZFe`UZlo9?)0Sba{K4GQ}YSaykxW(YnV69Il{i&ofvSD6Y^ zBYEXUigAthT~k#@{4RH})wrrk@d*FbZZZx%H{`G+x9kXI9vf`KCfJM)ptlRz3>~Oj zlk55y-YFdW>>>0a@DU3^UQaqZG0KIb7lf%FfT`;@Q7120TXO1y^YwY6L-LDwV0}3< zzirU?aKF1j@_e4XUL5Fb@qBxFdOAKXywr66mV3Q};`)9fQLen6?D6J$59@p-iXXkZ zTk&~xz5rORFD~?ks89-UJ7wtXzu7GgRO!?m_ub>m1(Oa&dj6yICN;m+krTpO7)`7H#k7%Ew{BcXH0+EStpQWhj`tP$PUnrPzajAn^Q)7Gn z>lMays0oGbf>Ji0oFOX#5?E9q#dhOAzZ66LMe}=VJi?}|Pum#tNDu-~2rHZ7bqeK$ znpd|R6N_#J&B`*7eD<}+Ug2#GdRM# zc(#b9kClEeeII$V0SKw&R>`(?^bO!U?s0I2N@fRRTIZ62;1Zre8OK2(TUUFpS%$)1 z@cB|5`xKNayG+{7$v;$It5uS)Lw)kuQLjXM2iplU z6?k3@yWu$GZ6V5I+`b;Ls$@8{&EoQ{5hz{Y9gb;S)5>n!IG(M}aM&dc{fgQM6UW5# zI+kxB_@X$QY`U_pRgl!RP4&9UjQ47JoaR{l?fm)l@me0ee8R3dkh(_K7+chE{pLop zDKp%YGETix2~;?(62RD51=lbA5R9Zq5dwCMB|`zh;H9he{1#1=KKH-_}>+t4v$E14pw1(^Qo{ikv zG(d!*5I@)?hx?8llB~_o)&KSz?vR~)p`OQz(61txUSufEC%Hu&LOPPwIS;UN~t3_=BgjRkozV;Rq~E@FH@i=v`s zJ#~EG){mao@HTj`&}E!cNd`+(!*#(hgibz~azra1#Pzt38mB`WhcCtSRjm!W)6Ebb zN-C3%-F$WOjQA9y`ZK<(?aT*W&D z8Oo>Zr_8c%R?mzxDmZW2HjIH6`3xZV@p90$Q?UU829u;@0D|S%u6P9QRU@|GBSe7s zr<&I8!3=GOFjq341ZAVc)|2|g*7rlMr5=4DW5CE-cawF_!c00kS9qOp5&Daot1wCu z3INQ#$3^xfwRd;jygNvAMWdQ~9+K6^RG+KJTKl8hgQ{2B!I7^^EFQ|Pj0h4Ix+$Go z@9%_-zu|;biDiOC25p5xaF0b>A8c?9jc4iJZzgT1b|rV*6yf+UbUCA6H3%baf;I*u6fc9Ml1>B=Ncm%mmMHVO$M#5Z%Tp{e#uJBs*PSi0sOl zTF2xq9^Lm{fS?o!1oJrI(*)f)&7ZA(`72Y7^C+rplS4u3@LoXJ7bKnM4b57=h5z_v zF}6iLe+8QpJYwzYc226yT~giNA7}+^3y9VL;Oc^bNu-Jj;ewK>Ol0VxR;!EDWCC}L z7g!D^R+OE2kouJBCZ2!j#&c3h{Ps)*>6M%`K=GDfV>hS1E5lE{84-*ltt8L~r;0=?TD3bA^VLIlVE^5lRvR)!VnA7fI-jC}fz zUa$^AZ3v5e7wJDP`L`zc(vCAKuueDUIJ8RoegHj1bnWHEtn4ei&ZybzJ>&|}9W7i+ zTF7k5q|X&XyifhFRRR9$c?it=^f(wzUoRhEF=d;fsntMvH??t!V9dH64{_SIc|kD=ykDLIkw-fL4_kzS-{Iy;C`DX}JLC0vRD$3) zY-g%u3c+`u<@^fDTHvyeg?fl@{x9~vd|KA2$b1vFP)%VrKB22?h%Bcd7n_C>QJw_Q z(U>gC%Puv0aX?#G$OF1L;Ft)H;{L~uFr5&~PIU7+vMZP~7dEMK^3ZqkFP77+SS0m0 zg=cHo$7aXb!-*InuK23RQZ+5tPJFfQW&0JagC-zVyUG162&;71`TZKz-!nW<*DEdS z$`|u@hDPxdF#;xSWG*y6oVc^cpZEFX!peuX>Z<9i>VEs-p;n|9-oDc)8SG&F6AwiT z*{SQ9#AdcBvAHn5#w`B=NhF0Bd2Nm0%$<^PHSA2#KXPNrJtFBiD8>%8)YNQ$dlzD7 zWIH*f>pRtUSJrl1I-4*TG;%g~dU%f~PMY3j8EnuhLr089#kYFE{`~UUj4z+>x7~{%X_}lz;e~YGH!@@Q&ZIM{(zXZ)alRB4OV;~551Zl~VeZ6|8 zQkfb>;({3tH&yE3M&P&6^I~VX;4%Qcs$lBozI8mRO?9b3`)7rV{}-&y3bC z7lc>?R508KQota)gNEy_9*=<3j(=4ffXtFI3iXtsJ?M>N!?(TA5Ct zxb8P8JkWdPXMa#m7XQB_VD;C`gUIt1$#3^YbD57v!C(FL*xh9 zx!A*>^+1&Aux{~#GO~QxbiMCU*fOxd?1X5&rxqmW8amXJ6gvL`yFnYsvCA(mmT=iL zR)0ERmTnbw^Un)f=9U$8^`9GB=6{!lXn1FS9)0^C%XWY5*wl})JXYP~XReFmzfmjy zdp!uUYOLNOe_AC=#;bcYt^aS4#UH<%r`vy=rm26b{^uC;-{7C|e;50^kNuM-QQXOW zWug+MbrXnw8)Y%*ceP5IN!+K|+ykSIQPYGNS&BL$G1vVBWOYq!E9yd=mQ93$V$1Zf zH_3UZaN7Oajz_We`e=-mujr`NZChiR)DFUWp+||G_LTx}L6Ssn5z{h`sSd=Mh1`cU z9qC?bOh&D124-CBRCi&C9tauKk+B0qZ4*eEO5|lup_71Ka)b&+8SG0`RZgPH#wZv^ z2At5An0PnrB0+>LEulF#RMU`^z99tYB+P9;TBSx2Dk>Pvto)=A)MeH)P z5VY_Zm_x`Y&Wk*jA#r^BRR(%APgOj(6?XvLH^wZe<;8hj^YGb%N?I!CI8pdGy--J~ z0n~X{^GH|;$2b<^G$-Vw*_Kur`Ab!sH>#7&xR9xC$v2BktQ@?>n`9jE%J0U8j^Kcy7zn8{E{l5vxYOPJ+AQw_Qa7`z4HG=C5IGFMNZmRS zsY~M63RsAy+$hlbsku9ERYG>zAWFwtO>eg%dmyGJ%z)I0JwdQ7k!Q^fNG9-wa%a@i zv%ciaDKqKO+J2J)+fa_zM({G5uEjKZ=s3!=HFzdzJIl`zpO+4gIQ_n3Bxdmix3k`O z2u8YT-;HNF*---j6yv#8i)UHfq4s1+r_@_Zkk;j=4zv?s(#gfM-|#NXuipTwSG@ij zdr8kS?HLHk{+)Vrf2LLJA8 z<0zRVy8sgsvazf+VyJx4@l7H!b6)PvsXOz@dK2-SWcFBkfn~3LZEY4BV%Ryl{ZV|G zOTtvnX<2nX`)PIhbACG^K@F~HYF(_3j-Ka|_N{mPnVENdleA}z#W|#>$h$aab;eff z>0L|E34)Xm70+-UjilGI*`8UqP zd=5o#fX}xiHV@McSwD7@zO~Fscv2TaEX1$P=|lP&o#%z4${AZD=)(TL5x(5h7B#`g zUdR=+GdQmAi#}478ecotMqdbUT+M7!I^;k>DQH8?#}sqwkI+jrLMH@{NEjVPrwdw8 zGOm;58h;I4CB}53fXgOrvW$Kg!|`VbaKF=Q(LR(6>KLX~EM{kQK8zpU;mgfk?o*B zLXcpk3$9g*Dzs(gV53DNv6Bb4r0qik!MdO$eSB*O^NWS zRtcSKm69qOOrLNTG1$}Alo_WuKJ^=&O-k?v5t5$ZH%fx@aLbxT#&i?bPCZAn6iZJL zrY!)I9~(plvRuRq+Has2zJTGzoA z>z4*IMLZkBs@X(Zn>9;RQ|L>eF+<0#``+o`CVSN7xBkfTx~X@|sojP&k~9g}9@$$O zI`|q8SDetNF=Mk;e15#yU%*H@@uS%cFZ8S{e;@5?)+r4ND}^QOrV4P;g5%@ zL5{VuS!cO`b8o-L{gYtYJ=;eNqlsg%8HtmF>z^n7N25VVjmG~f7mG;jG^u)h;9sR!Ufvm$Zn4%S*FyOI;Zof&p+iz4MFS>< zt82kbBsFt5r88%>UpW_EO(ArDnt2Y5*iA0wgiGv}@Qfcm9vdS=@kp*QNffJyUoI$; zr;RwlO0I6!%f~IBQqz`?Qe`#)lMg^*LX5oO77t#$ZQtrh!jUyxV8SR25Aw={7(FKG z6L%McI<2$;PPl_Sx!;zAr-0t(FXqiDG26 zEH?8Bp<*V~?1SK}l6dMo9TD4Q;caNFSX}(HIm37Dpzrh}(Y0JS*P|Le7)*s|M>Iv` zFB!hF^|0Gbii^7N=g~@2-4}2Fz__`pCb0LXT-pbvS*!p32fSKm1NSJ5^pUuQ;yBuF z_Lo#>QT899u0vF{nFrfIA<2UdXUNsa+;&?fdd@1kYVOBFz&?o0#a3cM;&6VpXDo;fzsEer);AqsxdUK!<6&j|*(+eUg2lq>PV)5<^kJAg! z*vb@sK3Ia4-PhFJ>i{Lc##B|P$uM%Zs>L7SEC|m~!8MDCmOa_Nk?mxV_q`OzdxR$< zaAnlh>z9}ReEo505cO`<|^<(2Vfd#*Oq7BVAkM}kdnM) z8*=!Sx)@Kh5S!%m)fq{0?-*Vtz(L&y3SM6yzYFIGrflQ7`UH+9d88^pT`@1cpk%-z zeMPvUO|Lv?PO(OrFXuaoZXOzczRu}`>iL`>ay|5W#u4b;O&ke2&r%>jxH?03VR900 zLW1}3O^r&d3d-jopteD=8`vPjN1*)1R{CPd`&~z?Y_M~^F@LDChi!<8rI`ss;_Lnb zG$pu)7qR2B5OP^6nmks;6AsnmoIuai9v|#;U-<$yTlGN8+vcx3%v-3^i``9{7594m z@x%dq--_-a2s_f7XN#?uN#8>JwtIF+G)&jwu5G4mXlCu2x{K?bXigPzJPz+$nkN`R zd6K|VMsaka2sX;45P5>a{x(9;lxwR-)lqdw_8d8L_D2;fp6?Uwi+7g)PkN4ro6xg= zahSUdT8Z{B@p-te+)S@W)Gag*q@3|Ep+Pc3^C_7AgrTduHCza9Wd?E{ahKnejIq81 zDhINggnLpP@R$RgH8@JyD)s{$?jMw2K{VqlL zQrp5t;+NR3sLO-UdgvTC@jrv3sHOF5;R~zqWXQ%px>drysI+#r6K*b5F;t~wCZE31bY}G%MMNs6ftEnGP!q5v-0hMCL>(uWt%&8)Ro4 ze$B|MG?In?(e*+&)#c7A$RbZG(tn7O!&GIXgsCe^pRb7jJL}p8>kCl+Es=tCq9|%! zkE9K0RArO81Fou&cz|yGF)D$B@JOkr6BZuS{Q8Q6lZHgq&PlChb(!+uK>>WqRA!KE*^oojoBma8OrUGT#3QW6l$lRO3UD1#G6QMKa#aK?>t z67@E#3Jory;+d=ODw{MYdQ^9DXXVtGK7#IpnBpwbVTXS#B6URMR4Q(YJTw$z^UY1l zJLPNm@zJ08J{Ksivv=;O%iLK@qn9WL3BUVrs1BC-{>K%AfHcs=Yr7Dto>v;qo|++^ z44!9|2kLdTdikRsKy(&?>XC%wK3o?WFd!d6bc(&Vh{185Y-4`vkwoD>JU2zi4!DLG z%&qI0U~!)0#^9u34Z!U^Vn`vuveZ#b z#+9CR46i7?R~>mwKc)7X<`s4ZI9{H8v-RPJ_K-hZTn^s<0L~0QU&Of)b3{mNu&~4o zyRR9v|6hv_!|$ad^Cs%(Eq6ybMF(--3kwG{K;P$(I{F0}&e#~UQqYPjDpA}2FGP0V`gv^VHjf}^Og?mI7q228$H7v zi$|bT2R)_+5_&(yV4L|k;5fKAh zV^{YV-+K<_?_wa3RTX~yzZl}hlEy#(4}^Fd9eT~7AIoQu*n=zSX%b~Dm2PwlZSzIK z5y-JC-=eTiMs|{@yJ|@;Q#}N6Ns%w@$_IBO<*2HjR}NSR?DA!@sye|=*{90ZoCo&* z3m|s>+~!M{jMRV#zf(c0hb6+7%P{XJGX|J*W09naNgk!mlQEjawfg(~=blhuLr}Vl zjMg7LUFt{pS?EpR{+CwY;eEJUVe-3^W-$oBB9kF3cT?HYAC}BMj{PLsnkSUz$>FolnO$xx~k)slcH& zQJZ2BU$YqTn?z9{t%{%nQrhfCWWKFdmq{2u636@Sa{Sl4-47}JeFMd`?C~4S)t-$2 zZ2fw}Yv#AeZ@hIYZm$zd6OtPOUQ~Kx+g#(;dBhX7PoLLJ1xM-b;nqt!H#k**z#kY7 zqasHtx{||dkz?&}DpojDDBj^nA_+-PV?^6r)|!gEH7Ym=BK|Q9j_tfe{UvC5ZSA_s zG@F0T|LQs@ei=8*40DYFL7Kc%#aB#b-fbm)ydKFJY(F{T!TJx5IME8MHDwBgL-1xi?8$d*A=K#KIv$HS1dLQ^D3f|)?bF@PP5k8DfjUU zE(LX2H~=wj*V4UKjjje(;xD^@dA2<+goLc8HVh+Uiz(N`)4^=k^PD@wG8o{v;v^JB zJDf2cX6D46x~Sl3$>*6%K9KtLJ{z_e`tP%Jqxq|AvH-n=W{?$l3z#V;W~Zf5P&~`& z04I19B0upQTX?a>q9L3P0-UyrKIrf#m%+d*WEp|3mH^NcJ(BIxe|$XPRT@5XJBC7Y z$u_pPjps>5iR`)~6s4L@!Zal^)M)2RJua2P_%iZJ@IRv1DrIjvv^vBX>-zSZPsmiD z^3lPEp^nXT%TemJtm0kPCg`H_{eXoQw$}}{1Mu>^p)%!do~ZoFa2(_96Hk`GV(3X0 zZD9^QdSZ(HkA*8NE*g{;7;DtQH49;MnZw1FEh%qNmR5RLt?rtReo*I;ImhzXc6XL# zM-9aek?$Mv6FmiV{}oBVVBDTTBo%;L;g=1?q+Av4MMS!_dlWi%QXKQz2utN z$$JgqyLt)F>G)R$P08@cbc-Tof1BvMhR#hO{bEKSWJ|3y;=H*I79|a`ey5X_xrpcV z$%y1gl#+mVn=5j}U_3(n*%gU;Rjf$*XSxn`?-oT$VuE;A9l3Oq?zP&@DvY+YadnP_ zL2RH8mHd2N-DTk(Uuh-7F$T1tqfu#wPURG}jP_NRNV7Kgxg?`*`l`#f2(IRvgm3A> zf=gwtz)+2GAZP}<$aGS-(%PXhXyZ~~=!p_P%86fgepelHRH0Hsjvc$yPo|m{`y6)~ z)2iIKkrZof!U;Y!;hyQJP>qf2X;_QWT$2CZlV#&rX>!f`=)l@-8W~4K#}bym3&M*= z_j|6h#M^X~?3)a`uG?rkMj>L2l*+reW98Ep`V?Q~CI;h!5|CVy(iiI9vzw$mc(el( zDP{NGscHa-Fo0@I>8ZvKhj=2k4@*4#NGT$Jf6??x|FyE;$gwFcwWrW{Re4IEd!U`| zVj*0rpdxfcWI{U0BIO`_1h$T(oN$)cmvOoICa3#3&Tc=$9{QdI${`?9TH3 zBEH(E*tU75?gNhPhxCkcWp58wSRT3As79Sh$(yC^mY3A2daYcPUmKD~?6me}%VtBk zQP~F!zPvUcRt5CMiD6)F$I4u~=9>*`r*A2^rsZMdgabtx596cDxLoThrlyqe^gveg z^*YnZ36?UwYk=vcXdwF%du8NSO`XSRv~1?)#w-nFIeO)55+)zOMG7gYpwA^v#7Ap% zWaW=Op*&3qJaikNgZXYX>JDV0P)l+ZEZVG?74>cC#87XW*WXrOqp-}d0B5#NU-3~E z+V$O$LLMtTl=xZF+T`Gkd}_65eJo0UnZ*n1c?0pK=l6YYb`fd^;-$0f9mEq2dIE}h zrS#l0j2Ox4Lvf~^>19AL^&{8^oP6Zl4^~`a!n*0&6lO3vS1{?Gmfi!Y^(X&!X+KzG z$wA8JE0Sq5stg{~4D%^*83x*L^y8rcpI~x!8GRCqT6+*OAM_KzQ80D*he)>RNMzMo z7|1tU&?&*`kSL)qYeyeasOP}LtZwtx0?z{V4SpDQ!csKaWVs3q7egBnK{TO6Q0$-r zd1fyg-7Nw+n3ka@N%`NsqzCuuPxfzL@Mw6f0<#w`Rv66Wv(q$zzuKkygEGFqnk3=k ztmOZ?o#!WoJa45>KA!)zeEn}WH&Olcr0}l#2SeWZvBn9M6~>>P6rLG;PZmY~bTuyu zqqa=&`89Qx<;PdrKtOq?V-%&Tc(4Vf9A*m&<2~}ibZfjBhVJLzG9g&Q`%bAKHTh}G zw{Twh&joaVhpcYY;y2p-=v|AvoFtU9{!wX_wQ$Ssbcfp4g_0FP>`6xQc<}t$l^yPs= zzXMvG=-pPXrbSo}Pq7UHkntDOx$3LN+z8$t+siQt5R3cjlo%Y$fk}mKDg3bn!9?Ye z?|sbk%G)gSwMEO3QCDFkXT=4`_+EpRLxs-B5~ZW^ey$pTxm!8Z*(*pcq_diKB!X%5 zFd8a*AxPgejc6D-=-Rz4KQt}JXfxW69!s=cI^V@-rD8L3uYWsel5cp)Pk zCd_C{Q+=2l6&NAorr7*GdH+-qt3um0z4PYoJ?7fnhPua3BS*Mrr;tI2-!YW!21eMQ!&3s zPJ432bMA?cbITn?t2p|hGyRv;tzdo+Qs#U+M# zT9tCY`YvuneG-g&zi(=|l~S%5BjSdHT~#NVgBS?-1EtB(Y$FeZ8hLvE@&?{&!Jf%@3&mKZB^p8972{8* zU2=2S-#I!^OEoSOrBctD(v`Gs1)|o}M>OGSoucwDY-RmWW>H zuF70j!k$IfxU&+G2CP{{;Y^AK3j_2o-#M}XVVh-j0kMSC82pMuNMs(d8gcajL@pOQ zDWvZ;&|93^r!36wZz1_hXM(C`DhdgRVOQ(dFehf9_v7SClCOAuY&;?@6nG9uhl!{o zc774^M{ewG#2@ZV;Ad5qAqTa)V3LMpnC>1_iX|Z$h>=HM6aZctwG)&l)pQ6QWN$p< zc2`??-b7t+wq=ZT(Dp7o(h9H4LQ21G0&H!6gcGf^I82odL_}RJo=)1v=sfrb=aoUB zw}CbCyt!G^y;VE`n#gP24TQ+gIU&2mX&0as8<^{dDUPCSE~C|{XP+JcsM|`&=vd;` z74s&Imf4I2m6rHQ*tQWFjz-yC3M}i&M&mB`U(I-gyc4#nz^f|e@PF=O211SW#;br0 zfv5g@--yQe>!Ify@F)Sp1Uop`cC&oYVou(OB)47xJxHLV;JAoUZ(pRj8mBk3VQfL3 zdB1jwSAwCpDuFXGqIySpm-qplk{$>zOMjXV{O^qb!##&dVEbIZDwiZp(@1%j3JLY8 z)*=2RMz^qh8_K0(Foj#m`iuY)Tq^!QD%YB5J2z@*dmG3kZ4Nb*l$jZyZ8^yA7h^(pBi zvZIY|yTirtD)7VQg!^;x&ZomI52ku=%nYAcX#p{^fZ4np8wZ| zsQ+Kzw$`}+M&0`Uyon3C*PNn%e81fP8+`pApPt8aRL38q=<@%4+5fHf%NOLY&LG-% zr2c5IWqXlShGz*`8cTcc{A7$C*gI1aH012Oaq4ByMIr@=CaO@%Q>YU^XI}5Y%rn)up&=WIQI4Z;4Mb?>D z&zEfbI)MIo-O6spRtXQ;Q|@|kdjIO0E6-iqezFhgxu=BQM+V8m2RKSsCH}ZG#5>n7 zzoQHt2XBrc3a912;`5ye>vjLMBeZv<^09i=#~?ZaJ9H@qqpD-^-oa?cCiM-E4A`6A z!P?Ei4~+?RmoL9Tt|Gt^mu?Yz#jC7Y&mRzE!CG`No|wt_`fteT;X|;Pj0d3GZvG&V zLD7%NE5QXLbZkJk9cKKwiY`dd?w~T#M5?ihUCMsj={_?2c`B+crh4u$`fPsh z+0dDfd0%;4pPslGkW^3Uk5E!Jo;OeCy&Nwl;%6ChWkI|#-ip^l% zo41x9enHsObtWub*47e1A_&ZgOJPx%oW+1tteF}HP+8SKT{p$BDNXLZ`OUYv@ZH7WsEv}RFnIg|weJA>vVU#Xif$Z?=34|i ziWf6Inir=2jH=jphMMTzhv2&Pnly#rQb?y&1<}YbOth(VDFO=(GiVqKmAn`iGPYyj zB55gr8j;*;z_yIeDR#}*I+6ZPfGjm^@*MQ|X2W>V8_X+W@6}iScF{;#-nmyzE3@t6 zcy=ShVXbiF4_^$WK*jXBlz-*;aeT1KU14ALyO5AWCcqg%_iJ!)QLyTJ|BVon(a^Aq z*wJj#?w>rRz1dV3Zc^mKAs#5~KMbShU@ZT{ab9#cb0W%F)IJa?2yA-l6uB5GFW=g$ zuMj7u!`I}FByx@w2tlus5hF&!Cm6^zJ?iv#^=I?*D45@n@Vb|6_ts}j3?%c&9)Nlt zK73+Ay)v9XisSQLBd!3?A<82b9wO?aF!F0|qS3|m^YL%rv8+h#6hzAexoNf;TGzRj zF4h(B;l$3sqy}E?IZ)Sa_JZi0r8y$qjC$}^8JNB^fm7HObka9xWmhF_K5tSK7SIGx zd@_<;?@FtDi59si^FBP!2$u}Doa*VR6U>Q|%)?kg&k(E`q zCI+>`NU~{;%sL#4*Ghev?p71F__sM39Y-a|brtu1mybRwuPJya znJ)Xi_m!u50zo8SCIh^W-#XRs+0xW~v!5EW3)71C%>^r+$#q|2yVhjY+1JV>m#xOa zas@)Z2u=8%|N8ouMZ8mMYWV}DT_k20sTB#9mia*z%U>pdMmGzLX5*6vIcahx~^bGIgy*S`&PR1Hqv zu;L@mlLX;V=YqrqZ%RimRprtRv;|bmkd8$6VQ5&+_E>Zk+e%nnYFN=jqn7=f5PRjI zm>+OtG=z{AAh&FbJ+5iWBk-l46_{Us)ByOQsUvZ~CZev(Bi!MdZBXK5`iIbvN6i!Nvknmp^>5_9L`I_8es; zg~vOX%){}*UkaC}t4C?VrA>j}$k*2g>QSj;Ex@R~x4LwtjRFtoL}4W;yfY~U1R?14 z1xi6`TBwArSs4fF<+oJI4`2L%i25ELH!ZKqE*P4XIU?{*$6`0+E%ZdqssnW8<;Nl& zDDsZjJnDVI9dfNgvH&eJfhLE!WJF~)#kWj7^w~8`#^`UDQZPxo@HgX8=Lr9-k!>dr zf{REdsSk(=nBh5TDGU3b)-UX63(2iK1y7jh)ebh9<6 z{CB9(hOVHYpOc}_9#0w%EcWnT)^4mc!5@Xlun>wdTn3lqt(aebIZ~)zx((fUp)f>S}z$ zoBs-+MTc#lS;p6ODC!`#?49Fs9b7(()V^@t4B=wpzzl=60g=~SU|@&D$!Im`5i^31 zp0*-6j0dw1F?pt3TTlLy88w0*}L z#UA?2@|(55m)VB;_*bJ(vTD!;bCIGy=%~2zNn#<=3huHwpP{R+_W-b#Nl_)BrXxqp znq-^Dw)ezd%8lOMJemHxRF^x^jq~Yf;o1&mB z)+TYDZp;(5mjHc3fFX3(q9Md$2K})L2REzx(X%^{XCSzce$D$-W!8oHi*F4&d66hLkBJLa8{mZ5OPGN6C2dAM+glo&s`00J*t3OP{H42+P(UO z^ikvowTM;R*C)97p&3DrM{Ug7lq>RO+V{N6fRARu*a$4d9JRNWSyzU+gs;l%pW}<@ z*8MQP{W_v?>K;F>fYhg=j@4uN2m1YKU;e{1yj$)Sb)f&5nEC0jZuR}R|Fq6?E%Re7 zb4oY-@(aB3i}e2uEkAO8Y)G?)^j;7DiT{ak{b7_nf1JRo40FAH2CV;Q#81%wwZUzm z#@LR%QVr?DsQrfia%43at<$aqu@z$#^Ex-E5s3y_dM2**fv_UmcSQugBsG(lVK5q1 zmjkW|C#v2GIMf(xIdWu{3*N_#G7jbFMBlIKO%x`OnP4*ggRT-rjO#>H2ag`CeHgur zgWoIUZ?(R1{}fgv<3?%(`F3@v|?W?6j6eAA?(J&%9C!DIm`ts4Me85BpMT5~`B^J|wbvA3;@+z%v*Uy$Qnc z(QrSXvI*G7p+45JGPr6}BHf*&MR#?4VIZ&du4#`-T8k=p^Q>`_rDPCfZ8PyI4APPB zI>v7@rNGt$y`~|ay4uLjkrEUjJ4+pr8Z_kNQ^r$wg|WVW(AXZe%pL*_kzLgT=Q=yx z$A3BGmADM>l2yw^CP7pZpP#JTvlG|A>#HZ)u@1kXMw>*GsfAf{T!2}R2byQXuPB!= zYj7x*k}}+_X)xLT-<&jZ_jcu&3-i4?&GO0X<(t_WS^vo-)1y*_^ zL_vQ(Cr)DbvxW&JxYw@g$S)0tpHY(lMIR8+NtJPM@bk-jC1_7kYXO&*+ z>UC#Loq3t%jyTS!!DEhD>LZP}Yimk}`M`tOrmnxiUY(lfFGw91m4(wC7|Vpde1|h? zw0nHCy1m}s1@}0J>qr9qU~Wro!r`F3$^@928>kup=x!#)KVJ=MCi9bShne z--J32c*jwVCWAKN*eD9BykVjz3r?I;(yNevd0+fmQ^o7e8CU5QfB0hWJ{@hCDj;xc z?U9S{gQH=d-}iGEioQE#fB;zDd_r9vfL;Z&L05t2%0^xQB-?{e#Ks|HxlLrKsN}0q z_gch7P1+4MXg#H|QX)r4|0D7=7QaM|C}EzTsf?2B-v|;ViP)qh^(flOvF;G;@II?V zhbD^lcTdV<9CKdDUs=Juc!)~DD4ak-;U$-+v>90uM)M>SXF&-7jELNc;J&SDCrk%t_&-E?f5@!Zj8!-Hk;Y{5A0dGX_ z*u1)?=XOXh^S|nEg|PL9wV1%hz{bepeoHs z@4$7SvRjYbt7jdHjH&=8p*GEH6j&y$MAI14IbSFh=sNSJm5z9}gZ}qKmvvfQsB2F+ zm{pLA5dB2zb0s&Y+HV11Md&8J`G@#bJN~;<*6NVgD*izGLgc znm-J@7-iv@h)GrL-t&$jy$UXq9R{2JDspWA~a7dkIy{s5}e4f=N9cG+a2bL=jO~%3+cPrzoz}s6W~@IVN|$ z?O2%2Eer#siKpmaf8u4sd`{Lm)K(4$MJGg`^SoG^hIb^JjLSsR0_;*BCQ8cpQ4`nA zibapW?upUBvR^0vdQ2D(-m;9DRywbU_s*wh+q3DjKJ~dFIiGu$JsZ}p@A2qUREJE) z2YDs#Z-GvG^Hgh(Uk#Of(TNi+iPBJ)9eZnX8hn?ZBKBHS+P6m#zuH zm4U^d7@0|NMlCLSUc-bbj%ri8c6S|Zfr`dRAd`~^VHzVqW1B1Op|u~;rw z1l@lga|IcQ+?8Udan3<->W|`q4yiR6d#eVud!ghGF;?4Z&$9ah^K#W4ZC?q_I{2GI zp@emxqk?e{?kH>ZK-&llK~e<{!iEc+iEIhaSRo+jGSC_z?!br2b8y;;(Qe9qw;8u{ zL~xg7Fv?FVfpV!tYcMF6o=Fkn=9H5FRmgJ zjgB*+C!IwWQAdt^?{Hr%i^i|FyZm4m_uv4Eg%1+!jqI4AfDc{`YKB2LCmGA4-PhcR z&$vYA#HB`j!Yo^)HqRzI@;jt#^qm|Sigk$BFd@%=j9<>9o>kjS=BiJ7zndGfaj9DX zx;cZ*H<}?q1GV;WIr9!p9Hqw=`mtxK9@U;UfRN&q^!c87QCRAwoqs^DZabf%BAZ&Q z`$W3G(A88n3r2v1$E_u&Mzys^@Y8`>Sk!>pqHr!SpOmkQ@?E9Q{=#zTbqrbvibUDf zg4%LiwOI$-P~R^@_|VU_CYL{HHwk{IOi))`;jU@9&9Z7s|B+2O4uO} z_xeXoLh&#*r;o?=QD*CsbSI5**d?jxu=6S(k!k8uj-Hs=KV}=vZrC>+_=$`+pDGx9 z{F!X*bjQ4I{#>%)*Mbs;0Ay)u?v%@b3`Xa_fr6Kk^Qw|SGze@nqk1`?CQ zW4c6m*K+@z3CKI{{q$)|)a}d80_tO!+a9qsZ!8J^9)L?LU)9lgn`TWNotuCj$vWOb zn#cEXl(3>HOcW|gt|eMDxV-A?tH#kINJZkwVN2t}gp}VWgI<=gj@|jT>u7;Pl94TY zT`Z=};cmNr0D-XAo6g$|u#vFJcfWcJr)o97M696%ZZ=a};;gN1v`e(-h^fe6O8C8F zh8haUtX9un3csH#0qE-7$6yRn`DBJ^Fwk}tqceIiprG8Z=2NO5<<#3#+tS1D7TgWi zDw96Jpi~uOu>dP>&EpwmYi6s%pof8mzW4Z0o2w&+9B0;_aJQO{*Bper==(?+2eu6u z%@m0%Ea^-I*)c|(pgZ*}Bz<@d0|XUN1!Ldd3i4!PABP?5T64|bfJIXGY-gi%HC=ih zko`K*)gH8~4g4pfC@fz!Q#ZO$Xtm+NUp=4q)Kfq*+7oJ?j)=7t1s!P@aagPvO@u}u zJPwpN2kc$@KO#IPp6l9_2Y3<_VaB&tD0`E(dp#+tPvJ?Gz+8$>@*RpugJw^UaFZcU z(0OU8t4Cx@&`~?Xa?aH`gpw{1vCGgQZmk2iV}`~a08xUh7A7%?5VXLe7a%+|*&Bi_ z(wlB>);MZ}s0O{|9x4}nca2!7K35z%-nPe0oP}CO;2l5eJle9Xt2=zd!LAo*KC7q8`N4Y2=)WO0N(vl)o!ED*1n~Kiq9=izq9=m9?QDHNvfDWAtj^0E9q$ztV6Qdtd=CuseS~#~&dD4c2Ds+uq*{qX&J0)I7va zjUE^qk2fCJnCE(T>{53I?Vk_nM{`P6q;>C{GG-?VH@GpKm?gJ`EFVbCg>=w?LA(Nc zq>0C|r~qKYHJY1x0!b=u($!F%iZFN03d+agn!8q5h+tPBD!1UdMtgT^XHd2;v zw@r<%>Ps}HfHWU08Db87TQzJLTQ?YPKDgXqF#RY$DNLyqTbe`opo zzR)_7EYWFaDf7j^8VS6|6^Xf$xvRqE&ylH|R=bED$eTs#LAtGsW71ZVBJq=g3uB>$ z!Jxrt$Y3mBuyau94GpK&4cm=Occ3-%)s8a7lCCR9o5npNUC z`QK+Be)nm9{^zr^AD+(t?eD)k`>=oyzdir+`6v6w@6ON9&VKmY>#yHmz4|*p|MY`s zwm10Kr?Vf}tG_1_IY0mNzl=2KAk3z&)cFk_8EsyQ-aA|R5rTs#rqjq36sc)y&$Ku; z6&zFmKD4_;~uUw2oS8NrZ390^~c)VYf07xLp=) ztBt#ueaslkO%ukBmCtE&1%9kp&BSo7m9u((@%w$dWmlDlin!K9;(L14h63CPq=`W| zz)DsjNExok=@-A>oMKp3PyVRbx+!R-r@BHI6!N3lEL-(SecFE7e9AxFecGM>I6(Sr z%Zq~B5K8;+mW!r_B(bJaB&Rz??ijmKWXEWw2C`VZUH|lx-K^)kcH^Gkd7pFd^TzwU z^*$^Adv*Tj^ZorD;6{~fYIaqz75^AfbKdJA)OV8-Ztu)g#nxOwVK6|+-cXSj!&L-U zn3d0!h1)kO-M5#d1vz(W7C|); zCTGGg{5xd(&}xALl5b7PRPDtxn5aZR5gE_l3ybKm%3ryKouc6NA3;nu>lrDGF9Jue zqr{H!toNIqva2<`i#IRF@-@;`o{Kh4RSHoT013R3WH$(0>_R~8=0A8F;dY^Wr&dH!tJWi`@`&MJ@%Al^p|@d7}(@A3zciSHgL8mnPiO zqmRR?-{Fs(|Elnicbb7sFy{mO0ulUBfGM}79jP7>TC??@m9=|qS+#~R(GcodQvH@m zWl%!}DdKj3aaJ>T!~QsS0yq%9`9&g^yjLSX-(n=w|Et+EETaDpbJ!N-V%&bJFaHUX zA?6d;bhWhq&xYU9!onRRx^xC!&*Qk*!5 zyY8lR=M+d0C-B?JV()+*Q((i#?xC{&G=GN{!V+b#z0roQQKBM8iE3*5juO>TqIxiiO0leg$XqFE_A!?4 zf8D*CY-Euu0+8vDCU2<`TGtg{f)R5#b0d@6Z5QX6)mpq#hN<-Gkvas4D}c-Q8!=S` zTdA)r*_0{43KwX#rv*_>X|3@ES?lLbxB;HsusTO8Rd~3z5W8Q>d_k_|)?xqX`7xGD z=<%z;~LkK8B_7J6NJ%anwKiH6X_Lzz0E|;mVbF5DWhGj?hr!H!2xt}kL){KL(PlEFonLv_m`~Nax}Lu zxshStzB@HYt<~MUx|`4Eb5qzD$|;O_ar_ZSfNNQsbf4Y`YbMaXSdHV;Qn54rTj&#= zS*gzj6tqcxdXOML1L75=S_EA7P~cSL@@_2NBVH9E=4dp#;{`0R5UO$4MK|!6i(9fK zqrpR`Zx}zE$A;PlXP7l(is>zxUa0J28NgOIF&9^Q%XvXJFwelX^$yp(0DFSl%}MthY+eA!-C(xS_XWQ!6B4tCiMHr68=_uq~@-(QTB$^nPw4OJ^nw zcsm?~ON^WhqFT-?D|fpSjFFSK;+>TBYp&qfJc0X9E&zAuWK0y>Lw0gcv|%!VH9tai zY$y0@{-AKtsu35-9A9%5&G>4mpTiJzQCB;Nh}^@Qb)|tu`t+@0%GH3d2be4F0=aDf zd~Ev8MgM$V=+O=^S8gD<`^U1L#{GlR6ZOYv+BqF)%U=GuNhG3mJe=te7U)m>Bk?*| z8gC@*frQ0jf!kH9!?#NT6eGQ6s1}R0mR8Bp#oj1Zp?(O6k9!9Y#E)A5c7Q7*w6-g0 z>=cuP357^qJxD0itZy%0lBeVhQ1W1jtSB!q!@&?4-X)DHv}Rg58e&!x*w+;+YKQZejHH~R^YdajTmM2c+i%RDTHJyI=T7)6&3kll7YVZEpr)%-uxU7(fb{uag%p3 zjFV@tE?yoO0l0EghgJbJe)sg{ka;epJVIK4v>ZYO8f^PJi3LKaYrKB=7i>G+lGgV3 zyRz4I{qQf?_Ga`*+iSZ%{km<3o%z$@Ha7Ca#scbupgF%onXt| zLO>t=4Hvn=7)=}*2Of+9OzUjKtw$^NoO}?(4G)>OYcA;hvl(eAYwjJ2JFy~_5Yj&= zCs|{GUAVm$1SozhU_t4&&b2|nA<5n?Y~ECkr) zl`p%tWg?9_q(^vM$?Hm&Gnj4TOCtl7VGy2XN4~y%8L}-~JK-5$wZh>!x|GnBuw~mN z8V^-G_i+Hs++n&jJRjEt$Hm}|vKe8wOb|ZS`PzpF>ib~zQknt<(RJs$^{oQlY6m$!ZvJ|lY+EcthNO2GLl-*G(t~Z1oAI56^nc)X+$(u@ z9y9^mGr+?vscCkc@9`mf!eq{&{xG&2D?sg66FXIj1~gvz_?^()_#;EIGh1$? zydh^cIdIMEO|x8Na_i*@&J|y)CuWV#b*n?=f4C?dctxiwt!-H5S`Nbrn=U8IP=y}* zY`4?Eu3)O}rL*i$LkLqVIu#bmun#?}?+kRYKPqT4xL3!F!mY)m#NPb1eZntY$5)+N z;z1l$f})UfjGbzl#1K98Ex8v(flU4is%s~fjl#~YA&){z z&{=yOcCtlcg%3I19?1$C_|X7}USyeU#3*oSC!nzZz!0VVISmR9<0L!73fg9V%J69n9lg&pBH({|L=ZY>1ECG8RD!SMW#`3{9O|e4K zKoxW=f>mrQZ=E@-smBm8wUk^is#q}gW}JaQ?9pgNxo)dpnR~G?=e0oymkw9jD1Tlt zoPhl5Nw0js>n3kmMSgT)RzJd~{PDJ}vHYHG%YxQy0Ol62iBJk=IXp2XhoAh$Mr93; zYg}5~vYZdqmKq*gyxULYxC7~SS29doz{QHUg>1yeib%5?6pg)#fA760U9QNZw=y58LCJ+y$?! z#iWA8B*Mk?7m911Wy7_sY2iVvfbYgGn7{;B?b?_oxgvIdyTDj*jq#rt;yHm2uuvf% zA1JaJKxt{JTbNe2NZxw@M9A%=F!zLnCfpgWad*~C?jy|L;k#t@9b1S`g|MBN)(qRQ|ll3CV%^#r<-7{E;UtgHg+<#b*uViJ4jol(fs~WSj zK%^iSYqZfK!Ds&lXY4Rg;@3>Dif6svjaj!lD2WFyLpSPkch;`45%ayD4n2Ga1cg*L z=l7lw3=qj7DFRaEsL1EEkb;q`?_PwX9RS7?0^{4Lc;o7pFxR%Iw&uG-D=W+l*mlXl z+2bdGK!CeLN&N1`jPM0p*uUV`un_>NK|!zA2*>NHwgNMQ=KE5A8r2{Fq`R!2`}g^u zOt?J!^`kL?{n+~!54HeNXoIV>|D(mc#Nksvn%pl?V8LS%4|gTj`dKjm<*DsNIsH~k z>jJW4tSwQ<8VLtzB21Ycr!xqS`$)HC!DghEk`z*`&jD-nLJ;?5cQ3+g->61tpoWACtvJkW@{%4vuHb;JJ@y=0qi)P*Zdq?A zMgd{f5mOgQm@VeOSHnWH?X_lr?hlw~wIFX~wWUR2FmR379yWX}84)aFO3`XJg97l} z6=yPN-dG5U3MiotAPA-qR9G0NWLe1@CP-E^s`ik0+lk%}%ViA}R0`T>;{5nIknz8%~UIgMMQao_P z8J@zK%-y29ZLaB9y*k*d5pGFrjocC~NZv4YD+FSvCyXO#9L7J^mQH{%(1MSunW(>O&-QQ=@xWrD=9P185CEKp z;cWyi;qE|3C}O+>tJ`Era}H_NW+V-xI5}@ebZSi2>0lSl#3^ZN7U&Z*-0ieQ4lm9$ zH+e1d_^o|u*n6~1vc@!E1zIKE5T#1U)>RHRXN%#58+v=Vj9I#&w36g+ut9Tp-Tc|O z?qjb6Fax|2Ar1VSL%;_1(i{jy!Kri8KG`}(eTmjsK@+;hxMncV$!|L%iV;%eCfu?)E-%2=2JERvPP%>zs@168o zizDNqeR4&DD53qU-fK~t=kAZ9ch88u-@noi0s^)>9!SFNU)=!Dt4j=^G$u)mC$uMb z5(a;JVa~V{v2l3)H*|BG6RCE0&fmC&30|@`#<61hJ=pq~WMQwby7AZ>Ugh(WLIl*V zWNPNOJdvcszD~M-(&3AY8p=lYF|3i^0W0k-L19sYN?%%hoSw_v;xP)4dH^)FXFsgz|?*c@+% zg;H;zp)(A!abRERhdV&$Ce0v)rZ2uoIV`f!<;Y%jH6$sG?}iR%S{i z$bdPP2%085?qie~Of^g-Xheo#`8;V!n11dF>|i@Ea3Xw#Q7TEQBr%%4TNpEpo*#(A zGGz+B8jbn(uO-cH?r4=OsNg_rzT~=#GnA!#!AG2))=6=PQiKGq9tB$KkGU~F#v=VivkEcC-LmsmmUX(r~NZYSj7n$ z?!6}kB|sM9^&Wkt5V~41M)+2%OH|Y(=POr~%DQ4FHFj@;wdVOflxOvp0U9Bph(L;H zeJ9(9YeeahD_g#N))TsuscL4er`Y4OGreQ)SW)PI@5gP`VyNF((-Z0uSVLX{DkNTK zc_quznBuqR{eIg@`T%ieF+F@_M4d|0DD?hc@Pr`rAs z1~@&Zt{z0nRyg|$tLZuUR^h<9bHRC9N3`t((!M@!Yf+=Zy&z{7MY-|b>uH`Tl%a`o z)fRs{a4moJ;~<4E;^~GYh?2}d*BxJ5)DC)rIQ#b zL0h1w1ySI4cXYQ9mEMsMzXjmuy@>!5;hAKw4yCj!c<0_>kQq_DoXEo)8YR|m)!>4Q_$?&;0)d)^- zIK8v&zopeDP%pWEb!|3&xjz^7>_>IRYEpI@JmNrv`hift3#a?l$C3)I%YQ|AF&wpMLG}PA`~x~^-Hbo9lo+Z=+rOruv+q3{4h`>Kse$6dPbHtf zw9};ek52zgR`w{x!)!AFG(#zm9w1zs6ojs?d9%-7r?MH!87oS6=I1$(7h94NSLc+h}FBNi^$t6}NFvzHyQX zRoxz@nR`{u#{BT_WkU+IOzXU9ysrTnjIe;?$C~{p+zb ziu;8e8cXDhf}&esss+3M`~Wmg!suLkR9Z)mh0=Uz6Kkellac^4VM`z3Pm6 zP`p*eR4G#fEeE@AB!&$jzS*#?nv_jdN|n}TD)x2{7;8+fkxi|`?9Q}(0Mh^odN$4T zq6;TuTpTTSMvC)Uj;w$l4!H@BOKv8n-=3agT!-fc(&~f{OPRJ>JKcnYVFX-(I%HkA z&qP&qG-(pSM3%AXwgVPO#~n<;JNea~mMVpWyO z29~>F?$j3FVz$b1l2u`NNPAJHlw4ui0q~l!q)`v1fp$g%oSW^D&|hcEcf& z^p?`WReWg4zeC)oR5-zlt4s5Q<1HX&;&&`3D%-FvMI9x`5uAY(;TYr!tluUn=$ciU zq>Z@Aq_|~O4Z$UA!G8*#(bxh91p)^28r;8-l?|dEW&vx_-{A5zB8j#&6%L$nYhK%s zyi9Jljo|g}3CJG$c#)NQlCxV@JW+f-r&YG$HOurl@`RRr4jmL=@Lg>4lxUR%P7n@l z?ar%>XQf?<*iDiR2!HqL`h9RxjW7UA%|sctrto%R_0V&%!VuxmGI3~3OypSa5|1IS z^$OK2Arixe9*|lt$cvbJRPzah-vJ=z7j(;t7gP;{LOkTOLIm_&Ps{x#HjX3HncPzkuREwE(Z2=r`d;`R$rrK7GvN=3PXhYp0E_Jz#qu!<(RtFlzs`Uxb9Xv1c2*iLy~T_4{#E1cG?POZ!v0H!-mdg5BZxqFah5Ko2Z&J!93jXSIn{GSbL zOQ5*$_E?86Jr$~pwB^qD8KFhJkGAoTsGtPH1ad z;_}3#i&U-d!`VoF9F_G%B* zDke@uAg@Lwq?o)FfV&z2vx@PR5vZ#)Agc)ReVD5>5UUt>@oeRPdrjInP}M#RRT6WJfmBUF!$=3Ha`*w0bfkv^r#d()=%{BM z^{k_wmD<)sxTnW~dYS<9G#RNq9pWiHu_p=MsSDhcKs5sGGyvL^Kr{sFGzi#~Kyw7r z=}|!l|4Z8y5&@YW35@BHfS4wNFs1T~WB{fXUMWS-Oaou)LMtUnZND(^Qo0nB3b=G= z(qS^V%yeLx$)GZyHK5GB!DJo-ROzz?RGNf)mkOkGT;sp4HU3awNt1w*ECf=DV3P!r zq2vya!V~R74kgok7-*u&13L|pC>f4uB4!tfK@oWv9O=t)9EPa>u4I}c z5JbbULdjHz;D-hwg_0@W3wmfG?9c&_L(|}Qra%p)!VFD<7&-uQC8ckZV1*9Vd?wA# zR45?>5|q%4L>QrcfS{BvMj?dy;6Q1n z_C($>4C)hs?1^s5E}%~WTu+2z56mYGswctiVr)RkFg+2H{Vbtmh@R+xB7n~nc%EqQ z`AFQ-C^S!O`vY*#G+3VKwAZ}&d?9%r502;Hpqoi2Q5=Hj0QjA_nl%Y}#{%ob*w}qw zcP0(m#Pd20YA1oW+#6=+!LmGI1E=SCmKDqN8TPyv^(~kd4K)A%+|np%RyVXTWHC=N z-G$#y3xfD;=!5d9&pn$`3kfjY7Dxm6eQah9QF}YH`8~!dc?WLAGRAM2nhvdpe8Who zAbi5X*zJ+=DJ4f)p<}6Tk7iV9Yn=s+LFU_(V-XmKm1 zVjhP3a56gM2<4&Df;EO>?G*pXq~Jq%C%a_O8#D2|v13$1{Fkb!$Z3_6zqvhU zoeVos(~s7wn`C&@8he|z`>mLl6^ z1qe05wIFJXgPfBNe>BYpO4b#}a^e77;sBo;IBB(tY%)i+in|~m#Ngge0xB_q<#J`# znM1FHRq(=X!FRBbFU3_Q*A-Jyz6S7ZAbo>WgZseeY^tUefI}(%lR0YA6lv}NPUdAn z1zQZX^DD{=eON;)5{w9b8@2Ylg6H+C_8YEwX>5CunpImaXn~mqx`}UO&aiPt6M5bn zqqsiWYX~%y3$8Y}irhH5Q#=~AwrVP1weD$O0w#j7+$E$FA#**}5sGa$y|vO`4SHK> zK96f4rQ_Y6b;q+a*^`D-Q|(T-ZJEJ@>P=&W=KG2Y1yC<=yVhofNnGl;7SqoR+luWa zz9Q^nNm1gaAdu5l$Bn8kXgTth88RtDQLB}`7xfc z(j7S_Vb7={%yQz`TJU8Ghp305wq0ERY-CMmCY-Q}Gc9yc;ymqdSAEFna|q1ysT-Cn zGl-X~A)3ByHJLUov{Y~20OvV$m}Iob8jK#q_f8#KFN+BwEV;sbDVEPhdZFcy2vtMD zK%N*}1|i#4wR75$5~GKgLs2m+%epdcFEDI}7Fwc%Mx;2$YZLf_K#eV14N9OfkBE~` z(QNHLWlipRHt3vlS*IL??=x50UoxFrua%-J_8B zD-;qvzT3&qHGD1gTreA}5jm@h9mL+C#PlEwObT2(Q#_aLL)NF9H5G&_y|BONR=0ky zH0!X7^YZ)J67j5?;Gjt|y5{KGf_O03wqAhADOyafYpO+|)9|<^`7}GDH5la5HpY5> zquw@?sIO(trbc5QjPh-)X5y4|kAkZDs*+3go~Phf^t#%4+QDw<&v$_1x=hMEBPOEG zj8ID0?@Cr%T0bZHgqiCXCOk7KVp;bLZN$1s2MQ{Q-*6& z*DFR{=%Xy=DMF zoQ4JhLcLv+r)qCFQ>$_F+GwMSoOBrSiK*&gOws;Hm_&H|1hHqtgHL%ZMCm}XaP!o4 z6h405@5;pOqu1eWYctYEPnh%~k@kckNX^ihr^S4rz?u}5lAQJsi%&9=(qK^L#L&JN zsh)5a>8GCnF&vI;Bg();5zRj)=y_@cSVPc60QQkSbtG=$zhjdjLxID?I$naoVP69r z9s#S{C9H$!dftyoo-p;V_XC=HN=91H6T!$!+~hs`u@j(;h0h) z7B`3O{sLaJViZku-beSEl*86G#M**4RKEy9sQwdkCa7*Wp7hI`;M%^tn%jn&C<*6P7Gb82KdXaUdR2 zWF|$%i(jJLK*GVAC^D8Cs;0<$aDV@tx0W~jaZD)xIWjYXHw1pHWnuk9`-9OLfXh`s z$Fr3Asq!i8cXZTX*CG8LiJ^4)-q8bLqpoSTF{z383+(LS0?viGQsB{~TRVpsGu=ZfdO)Df z3-T%mKaZ+ARz9Rf(IKLYWa&+8BIG!&7XllO!=KDsz`pB}T9u?`L zYd-*f(?uF?!EUgB9k82riCP$&@r@?Uc#e$Q4coN|oCabCiz(D`t;qG)s`fVc4PAsp@Ae;>$>Ui)tum0K4VnJSd zXl?k_C|&b#2VO!ycE^i6qZRu0O|@(#O}vQaKBE-1NMl(~yAn)+!je`s&zgc(wo2*l z;CX15>9sy$^HDvZ%RKu}^$)3%Ra_zzYRIvaS@|2vxW8>}IZdr~Z=ebRG#l+UJ1!DaGT-5*i``{u9xrCv~Nh{|& zTehJcF|;oBmV!A-gFJHp+6i}}2b<(oUyiX%hoktPdi_ySu{G>5+&u^Guml-t$beYO zQWkQ(yDlq6bFHG(6>xxq1tz3N6cB{?GDGoT0cxa)UdgJByhFTacxO=BDP^t*-AW@F zVaT{`^&xT(#6Dg}@EbcF`o@34g=N)}te~2LX+5-bL9S#TSe#B6L}t{(7sLR+Mwl?r zoMGll*dr*K?UGgC|J6fn=W~@^RoH9|9OM!F&HWV33nx zAT7jWlvIZE)~A zAsds?(UQO#TM)(siU}}{J^00xeJ~2pcqs5~a|+Bt>DHvL8} zGPyM}H?|j4yAV9y4cpC8@x%MuX~o#_j#|)mJSci&nK!v=i`Y*YiczvQ%-`=NjvJH# zHNw?OR!g4epbcRI2p`Y?fcmSwiPt-(quVU{ccANaoT!He&{vX7&OD^68LuZM(nC|n z4HaT+g53xj!_9(R5YZHn{1L8s3K|C5WniltgXUT@hPpfP-CWc_$Lqm}1KsrQ5MGKa znRhkf@p&JGS|exn!_wKM&PVeKB1e0|K@&d+JDf@na0;vXczwg}l6chVYrMHv#Up3y ziow|6_JZve%2iu3(g=3T^t9((8Lwk*JPkkUmHVwwwB~BX^#K2`m-3CQt>56saiJR_ zBmMe0rPkGJt-7}*XkCq2fZhN_eca5n>Y5VlRzNpwtH9fHZe7SFEo@_~X_TVw$usf^{{6!I`z85AJ`r+7u89Ay{)c=bb3MqWYoXD6HtRCdj0)s{ znsJ*o4#zCM5?D?q17>;MXzQ1_4Wd?Ty$g4zMMZB zFDocTm&`4fP@vxQ^z7OBqSJNV#V^_}j?%SV@AMj4O~npgInEC(E4UE5JGzT91S_|! z`K_xTS5_=zxr0~4eavkr5WZ}jgYkMnE&=$Kws4hj^f*scTk`lBtnH`9Eio=(nPOdI z!6%;W|I059w*7D24o(zH-Tj`YnGSF%%TPx9ms`g1{iefvhqw_kfuISRcy@P`i#wdE z?dObK$=un$N#wW(|BWIaPP+W!!wGnFoI+zzrQ6isPyuGUElO7DNkq8j93`{^xpHTG zdgB1uw@{p74bG@)Z$!e*I9#|J3W1@WXU!ZYNJCPFn?c|ArCB5eb)GZ z4xs0lZK}G{^wew+xH0D9gm!-4=a0^u)sdS(iVo!>2M_asL;H)FHwvMan*0BdbjG)5 zdHBG4ZVu(dLmK}L%46*^9bBD%IBR=*4v_(tNsODN9P`k^o#z=|H~~fRx&a0JDGc7mJMloA5Z=C5VJ+W4k`||8dGAB>Z^(q&*jbIXe(Zob0ht%+T zqm=?ZBMOl2fqYMR=qh%e0yln0?kRR|P6m{^w2J%pQ0gA$OcpX{7poN)yxv{Of@f(F zO@JoWea<_hH}a0$F&uVSdZYcn0-v+rh$s3Fx3=J%az~w!TJ4b&6Uh}!^V<^Bk!i)A zlR5dT6ta4bQm+7fWp|W~HX~@6@=?8e2eX2}Y_Lu|Ka{$&Z4`xRZ~Ubn#?M6XWw?XZ zKW^ZR;b=hZ1}EuXVb}r!wO#>>-hP@F?3NX7BiMol-XafeV!gsHbvG540+(25o2?4E z9^ETZ3x^)r_YmYTbrbkcg|+~^Vw?|P&{lA{4=(G_;_;R{t{lzXWmB8C%>Y(rZ4N~J z70dC)pdk_sEqgXOtWMI#<;*(kble)z-5_=#5|;ir$q(YtWj?-b5Ub>nbv|y2!l@ z36!3Jmm@`J6=IYTYAt2(N-tKE3svj4gcLjKb@ZZ8a;DjhCA@Ep!`eacd0eXjWyKtd zjJ3%?&husFy?5}3;KurlbwSu-4KDTG(mLrD9oR@!3%X?|;|e2;s$O}x6Rp+J)JgV6 zN8~dtPuW(F;c@pIPyj)T}5>YMxcgLH(>{tABH# z+_lV&&&L2kpO|rA+5|{|K!fuvguA`ww}3jGV&Fq#=!O*~saW0w>N7wi;k3W-tpO01 z?@<3vIum(J ziP5%Bz=|U~?D-+>0iF>*Y`2Oj{^ZLi&z?NnQ}9QL`ilP*{MTCL?Z20+0Xs<`vcdGJ zvr`fc_d%b+7hp&K^1QX{lP{lq@#NW=?(-Mty7!)C{)_pua}od$>`*j4W2$4DzZ3?i z?M_y?iM2M9I7&8ya&^|TyS3*nY(H2K-q9JgosXHmBP=)+ zw=}orz`@_@;BhqHXYB)b9U3!U(Tt1r3#g|bi4%YC-%D%>euHU?iQVM=h6<@Xv3O|J zu#Y9HH~_UeSsAP|Te`ETeun2ZrCI6~D;QPy{Q${j#V8asx}yV;-{E-H(HPdf+&?4m zVxVeP;pQ~KAgfAl7lgbvM$fUM67!qQ(>H7+2`94D#AwED_i%sXW!IWQUiIWMPx6*k zx18NQxs#P1;<@IYb6jca383aa`ArCP`2E|LZ=aKkJU1+$QEb%|xY^W#(2{@0Dg_7Y z4c8J{!}C86Z^Fp9$P(k~62|5l5)J@}78<38=o)85YlJoGBi4i+f)d6yEu}2jz-k}o z3d0eM!F@a>2CTEQzv4skU|@H*`4}Js(RyRok=^We|AR<0Ffc1#Sl=mlKEcAEB^5k< zJFlB{|J1D=VsZ3_DNmx>-#UTiaf?eVG6-fN>b@c;Gg@WN%0ljjEDfPono5LCJO$@F z?^blds_p^3)bd{UQF9Q9gTH=4#+4yfdfSLNE@gDd>T&Iu3-WC7?CHV7JbvEq2`Sv*;oD3y zMywEE*Z9G$(0b8ZQKXp*Uz=Pt{m|TMm37sy8GdzF@tP$h=ANx+p|BozTV46Qn`sTV zgu69%JnKs}%Y*l&*?_PJ+imHV;OV$(*V($AE$Vme8M3Xssgu za`xTDyMMg7c>U^piM_7ocYRr8sL(=AOg*d);>N$N2KhfNX50#9Go@gnY)Z+DeDem>!) zn^dT0DffycQiDkY`NPB3PTfT)$&1cbty#Cnq}kS`95g=1i0*|9maJhH2a ze5V+8cd5GzWoO4sO(_YuEA>8t^x7S6yx;onR&DumTNynOc@9Q;CmV7{(OU{UWkKHa z@;P}WRD%(4b?-G{nxMp6(&kRs$NKjtGjj5ArSw0d zUa6A>x!jfo&v>n660u@cg{x)B%mYE1eB}!wkFqF&??+>b7L#*{ce|`W z$8?ZlCbwlJx7FpG!(e1UF_#DVdstYnojW8}nJf}^8YT^n6i89YhSOo)@BE?=gPlYIg~cKJUWMlN4k2JhjV z=Um+c2g;Q?=XesIU2N%3Qjk}Fxi-x@|AhuE$a!K38_Zc0J}e5k zv^D7-q8H!3yPVXCZ{J3$~u_$+(Hq zWsvHtkBgr}uDtJ<-*6EG(CF@BeK&Yz0ovIMySv^{%}B|%DQaFu$_vAi*9#1Cj9fuR zngMPVAukyQ7Uj<^rbKiF!g3uX_L>U1)}(;Dk}duTi#$&83=NXPJw|USFX)mNyxwW~ z;Cx}FLj*d2VYoLpnk@1-iD%E+NzU|dPr!GGB_+rX`t&546xQR(8di#hy z4;|>Zjp6W7*MgnKRbb3}z{qXb`KdMd`$uyE3rxO4crd! z_${`Lhi>AGm77&suCTuKtLv`C^aU$7Wv=4w>EBwe5=r$4|Xu? z!j!TOYcCZwF+|jsi6{yj>$+lW=3APc{QOcxYdGn2H6l|))it2_(8mDzc^2iA(|JDug$8vmROy^AUtUls}}g`&t-692OTu_9Sc?$ z9va$rY&A5p=45pGT}{dp%&( zXnRdMX`+~r(;@lvrT1$3qStprt$qK9s;sP6(Og12QUx;}KBc_nO!F-lpA9#N%a~SV zIaq$=A|D_1o=Fq#?Fx|QJL80LKITxmo0T4aQNC~gDLbT z_qGib!wvLH?#5r(Z;Qc&uW6ZPNwRnMWC8azg7jf9Ybi^B?j0Qh1fd9pSEcM-!^5XJ zX}yQKENiR3S@PO3oUl8P)re$jN_0p;GJ_@shq{MVv}p6-A~vb4wm=UK5cTQPr%w^A z@PGV||3PF$D9KsIxBVM~gptA@o;_QT|G0Sl4TL|IwL-9Ce5<~pLpT6Qr?3C>`t6$* z_Nn_MXpL?-J0Zg1X8!B%f4?9vc?HSG+#s{qh)f$CEM!e6K;W@xxmCpT!h_f(DQx+A zQyXv$y?P6t)tH5atW!n+WQdLWqo$&t5d)#aGg9yyMpo+UN;YMSwT|JiL7E156C2U4 zMaAl$F%BtepX}@_`^Ix?zfyNxEaZw7LSm5p<`Gj4Z@*?#!IfT}fcw(W@ybe5_@k%|T@k%?X z@rrr)@amStHf}!bzG10;PWx&&;e4-|63X1POk_b`8>Wn#i!PNcni`3xEQZ%zp597* zwloefYj+!P@dUChz6~9l*j?HYfzS<6aH2rbZHnKb}Z2>e*jSsSkqFHW4WbQ2OJpq+<`wYj_# zceKhcu1536x*}P7zb};9uZjMiE&)0oa~J{+uYK>>FX;svGO*QBv$8!;R&YrSdbvX8 zJE9_|*a1#8w3G#@9-V4Or`oT>sdkA@HGPsoJyxiFcZQHhO+qP}n$%$=SC!RQ&aPm%U=T6-Z_ZQr%{s~>RSNDE;Ez%rV zKxf?adOdo!x7!CpS>KMm+Erkii5QFbo;~s7V(#pdnXPFuy4vp@famaN=VWSN#B~*J z4Qz}y;6ztJL*kZCfpFw63b&cW_pN0@#noOfvh^a`jwi8~M0|!J4z|!cm=xSP1!@Jo z6-A1oE3x0=2*$Z;*vC>jLs~%_?X|e_po|&1ESik;tPIHqz>C-~`YiK~;<^tF9crY@ zd5njgCj#{an~|+)-)=HJr&Sw}^l|&>(ZVw^oUCPP^WXYsazv~G)SP9 z_3J4aeLhB!>GghVL%kQfw3P6Esr@w$bMc;$iUOdPj_NA2e-@_Yeqhc}Vvk}(0-IQJJYIj8G7iLn__# zyP4zdC=L(Zg0Ng8zC&Nl`Fa*JOft^-wiSX@fWCZ?yFrt5Kz(jZY8*P6Qa6S&S8pNO z$vW|1lW_a`c2ZMMA`e<}SiwuEb3lkq7hK$RMl>rDJ}~#Hpg-H`3n^=zaK^wU3k6c( zLWKN`4=%GQE#YGFQ?3FhPtaSkZ>{rf$u;d@{i;IKbww?Q$v~NsJt}c@1ez|5I+8>N zA|CNo-@)5#N=Kob_5MvdAn!yRnmJZo4*TF$H`0TSofWEnsq&`RuxgzAlbpAUWzXO| za9Ei)7dNcAhueu3_HbbdY5=1;s0_qa@QmPaj8nMX52alVB-B7>b~vMo8!32`E2>gd zDwHehEurGddiRnDbm3ixaeci8M{oZZpNEvvfs*d!YXz{oR&BgynFK(j$m76m!E48 z%87nt(uV8H{C~Wq=Y^Llmj4~HXPE~jX#cd^{d*Qqdk|3xs1Y%9XUZYIT z|5+gk?Q8XkKe>{82R{$~@FwldJ_PL#fmIq{)<&>rxb;u~G``R zJ;nRQ4unBWKZhUkuzXNW&b!hA z0{s&$-pp#LlMw{^e5$$oCq2`=!8N*y*&wkKJH3S3C&4e74aN5a@R7IRSaDVBuaYiz zKw0MDlVR;6WLe_!D>cQN*YeFFK=!Isx_@w+B*wM4{f4;)1f_z4ZJvRI`k9dHcP#|K zLHPQ!(3kZBTe^E##`(^^x92CFiV)%229^3gsF$tZ__(5s7YS&)AP$`7zC9K$zmz+K*o(veJ`=;Cf6fll#>H!%@+s5YK0#c~(UPd&EB$P)_@w4=2i? z+atxn9NbElL@^;SEzS|H|0w#aDg4}nEQWkNjv``JRw{)*b`_%=iSww!n8|RDJ!U^H z%jj5>+ioKXc2rW}S%zHoDcfRgU^ZOyGqgJJgrDtMXs<}OqoR4(Va><}qLdf~`Z{=V zBWFe;ti>GsBbbA@`!y?0rN;2J!B<^_@MI9N;%M(AJ>t!53?d;TK^AbAbifUZiOi}O z2RWz2Vbmz5LpeAw&UalUyV5t8$5h|dVB#uL47J8{iS^tbjJY_)0v?Xew-1~;Mm!BU z{0XPQ_6nRW&9m}wX&rylWDD*r{7ELsjTo`!W^zc!{ZI&vddD>sJ@|TK>E;AlNdRY= zj{dwk!F)eaks8XImIjHE&IAhd;G#(-O|IsB5o~duiBotVDQQY~4Tu3Q_6S@l{>mr| zckM2H-Wbta{x6WpdR!;YKf5xU24k)B2_i>=M#3=0s~7|GW7g|Rq|BX00u&?4EoOOV z8mkr4mtAY*)>|>yIn5+!R7}>Z_F%0iDOzX>IQ8uFsB3B~>i0@getN7V_!5@G>SK78 z-}ciCx=QzW6?aj&NpBG4jBQfA@_EaBRCmJmWe%oH?-mQL=ar<>5oeh0o`~~MUFe4J zyPH?E#M}z_MFS>!oFlnD#ki9;mDu(@p>s%CrWte{lW{>4AlRz1{>-^M-VV34Onl?v z{`3z#!A8)%4Zc0HgTp68kY|hmB(qn|V069lR`OYorrz@Iu#RxhDH)d2m6;YsG zeWZL9*1fHIHDyk)QSl&}GXLJ8kTtgbi9A>SR#{C<8RAqY^Fov>&|2^ zv_3BLC_)a*bd$0%XNu@Q|B?rP4}m&Gd-nUXJjdlSOQuJpDuGg^#C0I9tjZgmrl)E7 zC35AlAAH!1nr>U1$j`8nnyZMJ+Ba}36h7qWas!eTL)PPotlMU%@GKRQs_Yw7qn7X& zkM7wCEd%DEG;R*T*AvH}rPcq8ElNRoUt)_WZuh$p_slfwts_Om5rW!^R?M-$F3$Cu z1hW|PlLz!gayewc&{A>f9`HNY?_QC`ELVw07FbwEa{cr5{XRO|X1h=)g=`fJ;8JoC zm)C1@3%BTh{Cz)T?z~yTkdB_*J-rYJ@{0_IO{36~;hsjf%`_%r*8tSZPfNpEzA7e7 zb*B)8DWghq*~D`Pkwo;UeXv3WCDHu1=<`5FNET!VRh=&*8-nMFUm;pWS1C=1rg&*) z5^yLs~-788S~B#+!|2UpIqG=x$~ z=_tvjZQj7+blzpDdg0(G$yQq`@Y>RNLI6sK9oALCo>5!l0f8YTfXL^U7E$HsgxWZv zDvG3tPHVg+&|;_^uji%@9ru#SAkPHpNNWa&o+Z|PpL2Cf7?l!2Jz$m^?QWUD2glOw zu8lt0rx1?h%wSQFach51___oxXJf<&_f``Vl(GL`M$8%ManvCVACO=<-A4@MTbAf$a*_uY$8ohB+P4naij1y=8G6nWte4 z2=JbYVp5-|X2u*7%=N?9c1(e{f~Ca9&ZN;rbW|sB54Ju^#gny>R(&`VB@X^!!u{#f zuu&D@3^x|o{sFTPvwwUbi4u$$KG-(&lKa2GN!`s)HDm5@E)_feTv5ogLy+EdsGMf7 zu|n;JV9U`8g*!qAJRJWlo$9+H_66}q7X8M42trZd(3Br}UEbR9*o@^*Au@|EbqAsb z$V^@u2R_Z#(hAM(qI7Q&T$Q6B-8s6?&ESWnq;yYM@7)y_ZMwiI z%F__{A26-jkCWK}SiSRPd4h9KqTzq$ zOyxbaP>SPzJcNRJ&W{J^AC`DYeb}d%8-et@0;COOJ=gxUY&{094Y}~{&(z>Y>i5&{ zex_{#pae8HzvhK}>K8;nK@>kyku6L;@=-e=e*@4;Q4aqxGDt=41Qb9KsMNfOz-gR0 z$8%v}uk=TrERYz0?D977T2enZc~M6+s7|+oN{Y!tm$ZddAQvOOqt{APpTzbAtSMbtS3&WlirIw8aux#qUoG~lqdm+1e|ChEGF%LiUV9%< zsEVnFkG^=rv;mtnT^s5Cok`qCX!t&krD24+^E@^q`IIRH%ojUOwfX4!7y+7_ouazG zw?VKJ2~<=zo5y<%(WXhbSE-JnmAzX0P5*r(O>w+hI#Yrkv^~j4o-agcHdLs{8}Ypw z@tk{cr#Rb=sC%jMh-r8=P2m1(!#5zNMGD2W{3;Wr`b|aDi|{Pj@7ot0vcCt%huUKd z%r)()o5@woLRl-!LfR*KD7sr)d%s~emCRlW{7dhmsMRhniVacuDXaX=_hTCY&m;1aSu%Amr#J(tYQO-8 zwc=tmaOl2lp_rUd^r*f-Y%{`AoC=rxMs<#VZ!ORzlA=wH_PRT0{o?R-SwELuMk~~Z z1peq5sL42PyJ1YkdnI!HH3=J4-{@lapeBp!g{PG4STNr+ z(PnM=Jb^fNs#g^Bs>aili)J;^>x2foxILcUXrzrI*#$M~9g<&(n~6aD&vn6q@p8~x zbMU!XmEBKUeb^gV6h+0#(wOefuH0!BvWfuqb6TK@*|_Jh2ka>mN|%qKM>OUe>-d}e zTVW=t4YfC8mqnmcPJLk!1=-bzXEWoEybYzhP#0=MS19`c8;5T3Bpd)pH!-2!^%WIvk;)v5aeqM2;pt+@;gL*QGKYsDmjV(w5Pi zEr!NLv6VmMls7n({$K$1TaocDDdJpbMzduoi-qm8DeeF*)_XtMNE1nw7+gbh{SSqX zsx7n|Fgb(+x*Q=@3))O7`aH&-TS~S_=4!z-Vb$4&GP#5;KGeMDakJI!tF4f}qZVfp z`-7xRUr?To>kq{LiY&odIBOP1Pa%Lv&f8ZeJtuyrfmsF+FKlhh2rH+I7gly7Y}xmz zCuss~FcNHTbQV}OIO7E-k0}}1Gn%?HnBJ=QPRf^tAzd}lq^aBV25X}X&fy#nXPnFJ zdMu8AP*j_&9GYt`Oj#$>O8J!qysYgp`oNvh?)OzA5GYbe&s%2=VyeU8BoBGo?hm&6 zTCnsGjkaA&ig>;b0$-Sj9Ih&St#|B%X6vSjeT9rp6jRm3|GX{9kSX-Z$~2En&P_=d zQ7b?R>=!{)zvs!4SYg2cbC=Fsw8z}GX;u7{8hNr#z_&qwt$D7?Yu*l6CiCzXi<#_? z^i?f3I2eH`ezF+G>ycGqGkTbRN3I@Ou}weS7&oB>U|h57gZl%w#I~`3yJ3lSh)g1t zGu;couZ6$yXD!GKZ4+9vWk$CRl`TSaL_y$JhV(S|g8dnwXy1d{<4!0~#nm`2zu=XX z-L%+P)qwJF_f=cfJf!}>fr-2KN0kL?B-MIPa|pa^HLM=8%kaiOhg0l5dbpw+tNG}3 zQM%e0|GChZlKdn@f<|-`&!}Ac;Ox?0>Dqh|RGaojctvbEAK#D7Sctg}BgIE*uA`;& z2h#pCOL(@Z^(uwuP`T$t9Se=W+V5!fEeve*1{sqMj^x~}4es!plDbpj3`;_(;g!Zl8pZp=o{uMa>1J9%q56fWO@ z(x_v)-#v3M8_>`%G{qC?FA}u_;@0|pXR`uy(x%z%`Zp&GeH>n2crh*$!x3A7UJQ88 z()1RPS#ehz35-|>{#I9x|H*CjAnP2zgLngoy5&zOG#SUACF9W+YO3NcSltcuh3xlP zcgY3FpOp`T&3H@68OAn>DAR@YS(V#s<}FXDCr1zwAB?e$;19Um4O$#3-d)$Oy9D3Q z^8e1Q*|Od6h@yNC32LGe3A;Sp7XPOL8IJZ7w5=4~i#sUz#+-r+CmtW{d+@tP^V<$`Y5Wu)lk03EwRS3XfgZf;W;hYB zMn0#jW(JnPvu^cXxvEJP4Fqseb_Gnp>tX7k%xg=!0}=DbEl>HuL>-xnQq^@uliBEG zMhr|@=S(+hFcbbb5XWDohUX8>3CMc<3+oK0=j34CGvHdRL+v=3`=zaZpQveh*^ywd zkZ^Em>vLqTkW_`Y&SkEy%nJJZe_(rrV$W2 z1%k!B4i=K4Q)zJU#DxfmK=xo6_-Fhz-D0M%X#PzeRo9`#%oFj!Vk&I*z56~3qAKNt zvbOFW2q-0$%XdSmxswZ0c=v|i0!&9Jy81S?H{WZLrvjh)+XZevbvktdxlQnpvYY-|Ao)UUx^mS7HI$t%8VdzB<;)qKwCDoyC2j(yl7)*J2zDU0*J9D zB#h*kqU1R|T-x{Ooo>2`!6=yZgKc%J?jLF_Y2kbROqnR<#2A~h+(#(QBiuA?{IQIa zD$jNj%S-jOHDWGrgB5xkMrOe9vNoR|Vm0?^1|e2q2FGYj)1*)85oWY?)sfz&lO=Kv zd%-08Lfv7qa+3Z{Y#AO!zK7s&{k+{vi`RN; ztSjD=I)gc7qB9K=olm>d%2Bl*pG3r-uoPY*s-hIo@ z(^33e|DmaBaYAdl43Y~?v@Ox}}-uJE`VCwRPbU;*F zVD-e6R#1U1!7)JccMaKoe&ENh3d=pUQ8FXn?rtzlef0kEz{y`ip7yp*|5}#K)K#1s zim7v2)xVk}6M^7n9NC>RWaMYaPCr@+Naxrd&d~#2!~46&)VjGNB@T;Q#)SFJBSfWa zKbN=(P7-E629|MCtOh>q6ONEQgMoqmd(ZKgIKqUeB7!v}&JXbpeF~-= z(W#nkl#I-F1N|p(aV(SYH2BU?yf`r!4g;tW`LRU1HHeA7f(N=u+qv4R85NF;(nQ%O z_2*rch)M&Ulka%prf~L5$IkVG>QU$T*ZE&tqHMQ|C_h@FzO;!+F@k~lP)d)%*aw){ z2JUs1nJ3&f1E-2K9V>2;)CluA0qsr`{48gNKyFwr-_jEBkpg4*^2`KGC#NalV$=~L z$Kr1P;?J2pOwgpfK666v6xzA9Cu$rg@{({9=)2Jrb+xy+n)qV!u|5x!v1 zJDTO8qgr^-yWy9CbiSp}`4wdcUhH$(Xp(}q22GojWAIF@r(02qdSPbx_dxjhFh&Q= z1m@*AA=8w$5VTc|JhT_sXk${B&DQzXd&JjJ=um1?ci2(^tAL+R4m~_PeA-T;DxdG$ z4F?dQ?E?*LpwLXDO5mq~qU*>0&}fp6_w;yqhpM5upE?)K5KS?Vr@z3I6%-C?3#cvd_}{Z)0DZtwnc0dCmI0=EQGYK1XF%=v8*yh<>t{zmrw% zG)CUF7q;@Wb_L*zecQn0NgH6V5#Cc%Tc`lxG2j%ea&$5oK0)s(pPzQ+niP<}NUCui zF}peJt-&S4HmJ)$m`4xTpJ6!rBzdSs_~1?`ro~kEZ|M6gbg{TScgWvzyqUR>9atIj zFYW0D&>4{eC|kQ73N^Q6-!u>{AKaEb0yY`J6SHN2jTfjDFYd zote_FvR5G8Dv%S)?DKSd)yt_2Kv?X5F-R}Ja|Ct{;V+;O7GxL=+_3N+@BnZ9t*jTD zGxXvJ+3eqq9Qd!C4`Y8FL+<1ehThHXXF~9dJu795@@8%`RSGWtVWw^PPNDh=Ot#CV zrfAKL-YYOR)`MNOETiq4kr`(q&*#7T!|fLo*z+)wkr|YsDVLh`OkW6t_tE8qcH!lC za_;%;^!w8D_PMig#58u3z9ClHWfS@6%c+q3^?}c??;wp_6Zf-&mnW~Vj_tA`_%KWP zkoOwWewqdTmkR-$6yT77Fh9O~|8icI?hTm+6K_glch#m}I;F0R1wbYV6K)Al?||GT z9aDwLhzeD7a<3||LPrb|pCso#oXDev)mHdtsULjA^Kg~(C&$&%UU&y?IkZvmk`6Ud zeWe+7jO|hztS^UzZMEHHDN8|^jf+8X(B}@!F`4=MHd+`31ZGZmQrnOWqWJxQ*;r}j zXTXOaV?~+4pm8)#bCX;rC8e2<{OA1nueV~M#>X$w(N461E%G3~>Jv7YA$Z3?4Q6T7 ztP+F!0HTb}(^*$2ZI=k0Ih@01Zh$t5+xFF!P0qEna z3sWqh)PU7eB1nG#Pk8N;X#y%9g>VBGH`I3L8D(qiayWsZHw&IZ+K9yshpZr=!Fk`O zt{b5w{t6llQ14b}C_(h3})WF7vw#%NJFZr;~Hrx`~^{ z#Gy%X!Np}m^(lG*?~5mLw!63V#T?wW@2v%jS+$NJpNxYU2vIEV}4sfH!DEmBYP8uf-74V`34Z%M&tbXB`aF7)Gm9 zkP3+-&WmNo1nwt_t-Z&5zE(Se2`hx@9;HoglqD|gM_PT%WoCL|T|WjCI(0L~EDu(} zDFFcM#AQMbr3~AnrY0^aEWX8*avwQDnq$F6Z^31t1fox+jioMe0%1fZFXbLBMr!ZeA;W~_ZGcTn7G-69dV~d z02-c|KobB1Qw6*YPRmPui4cJg(=kO;!;cBlPYR^Acf!aae|yGyWP(hEJ%HWat_}ly zO*;|%x(hIQNe-EI08J-VMQ~28wAL+5#&sd5Ewto5kJHK3^QGNw&4oo7ris|jvqH5q zozGgPW6e*Jhy#PF{CL;8qWmj*C9@v(z>@xww$Y0>l>Z{Qgk*%gMl=Ke$kDNCZ z?nw-ap{~*aNMpnE(hm#8C=6!DxlmoyXwwnKwl^>tEFzlp@v?@mG|5cwVQhB>rS<0{ zlHIHH-zWp6b=>kr9x}QUyD$lHdiksBO~N;Qu)?-nd2#^nN&r(hqt|z-^XWhCAqwfi zSom@^Jb3}JUQsv;R|a!-?jwL1Hrc zZWxe|*(xg4f+0GIAqvslHnP=r78G?iHS3xWvks|U)%W7iz$K09U)it313Oh7mO3;A;8brJrcpdzy6%K|DE2Q{^A8FyqU%%`zlKb%kG)Uf6O7Im>L7 zC@H!1Hl9E?X5ER#mBd^D=c1b3N@!Idk3G6(T<2-QTqWh({Y%~?ITG$)!$Mn-t* z_Y*}m5EA085jN=M>845Z=%6i0&4u_X&FK68rgS2*voN8I*ZV-L;Xf$YG%;BWiOH-d zoH^rCB6ES$-}GvynAH2_%>kZ-l^R2^C(Xne%GhxzL9vz7 zim@qFRJ%OzpNVonZci2kF|UX&>R7qxo9hu*R#SZ6=jyzXEPgPteK&mLPWYAb)mwba-p)A;Y{jMx0<%v>Ge|}Saaex|p)Gj0<3nb(RsS)%c-BIc zm$kq=*qEd9i-Aor3&+|}RP?gI8%9GfWbTq>rweAPxG4z|h1`>!e0w5H6lbvhb09%hJP(R1ewk(LUyV+%RN{LKF`PkO^t z0}a$)jZ@?IV)4&|ziQL#X8mWMTff`|23YF|_M{>u#`hp<@wF zgNrO%kRzblJ05Cf)SY#zgn9Ye@C4z&Q*qhao>G)A{wVQQlPB65{t&bc`e_xBfX^8eEqG>xI6^47==Kvm&1lsV%T->O2CvqVkiDBFTk^_(&8Pp$lXc+Mpu;L% z8xvRM;D$*vp~bsfK1YKDD(!m`huV?hnVKH=HTGU^jw_V(!}Ldy-VUZ}6PdK3y!`~r z{|qM}E|A6-GXE>fQthMD;M5(}~b56oQ^rS1vR_Cqpr!gv1Gs?#+N2>xms#+@tpnO9!)fL-8G*ATh9 zuC|?D?wR^ZT7Lxha>dD`y&{}cCKa=B3MO_(5~C_g4a3ln;V5f@ww4!z9SR-I?T?Xs zY~!w+h$+`cLk2-M+nUzpdgh>}ghOs0kYYjTe#BKZC}T;vqoBc)KqES^(=hMm;4&Hi zHdOzEa2QVSbr?#}Wlo;8b+#<+$wW15L+8-8nv_2oOrxDkm`{X)#TmH~V+q%=tQ^$f z(Nw(~GZyqKo1f$rQB45F!DRx=;DXLl>NfKEti3YLLQ>1K)K~X)3zuNrUwVE&t<*aL zs>(ZVnhoYcPfch(Ksk^&XDNgo1og4L0`qCLcxFAW?)*;#DF=Q}Bm-Ao4>r9XM;prc z;AI~#C%Oj0yV4F#?=4SiGFa(0j_xfq6<=SPL{7qB$(A5G@0(Q5%*?N_MuuBQ-5F>n zF6ue{c4;!pWsXG8T*C_w48U_?tsW(aR9^JdYqHpC@B}d%7I{2g-Urf=DN3)cIgS%= zxVkC)tKJZ^78E0bKJLa80h_FK`KH*5G&ILQQu8l(_@&2VQG=DM3>Wn}gY!f43rBWb z0BGeb6E4ivZ^J%Mqh6H3e3gL#kn&4#!%Rr^)eqsYhJ~Wq2fM7|p`qUH0Dn%y(~l4Z zsxX^E1oF*};lk(}K6Xrd-SmI4?O2TWdxXr!sr03GA)BSW54ozmh&r+aNJKrY>HG|; zK@c4Mfu)v!t>&Ucs>iATqh#AaNSYs*D@@Q7J6(4bz~%?Gg5b$p$p-BU~wM$cSeZ}{~_JM*6S_K zGC7GnNNm3Qj-PX0%h{tr0RVFs7umI#fQ(Ojxyzx|)|8OHTH~V0`COe38FOaJnsr#+ zl#JTkE@Y|7jfj0dQAw#&;oK6?Yct%ID3?PWvRZA3>0cq0HJe@Qwqzs+if6d^SSb9l zU0iau;)6mCK!U%@6<2u{{j^vG)4*0D^}3ouf{1wVY#Xu_Y>G-PwG*KUqUz%ffU1CL zU8GHJOkgfqkuYbzOG?Gz#8l3n&WmCKIQN3*M@wOA?e!*&v^`oeKIiz=nSsIzwz7z+ z1FOudWw_`{z=ZUDc`z1(n*k+wkKJG6y~CCBX-#n+8YcxSdvzVvhp>T!>dmHgiD!(D|*>mufqjkjGbs;H(?903K0`Jkq#GPX# zijg@ydZ*W96haLbD{)Ltl5{I_P9x=~q}fm=B90Cr5X^KLB9OHegFxam$FjDXPQv&} zSlR+z%xH=0r9UX31E1Ye)Fv`Z+RQdrxA&L6G4HEN;71RM&v~`ahEATJ>2E96M*jl~ zZvz>GoC^+=qGgNx0@AVdQXJkK5{^n?#gbdB%#wnm=h7L6UufHqjx*2gj1$l zee*w*iMp~l^{tA!&Ea(Hp`X2*s*^Nr5SDSlFt6j*5=?MYjJ5(1YC>yxtr+4Ox%G5K z&gI(5uYZ3(JqabtExVriC^`8jQ5QnUuFhl~;h8w1(Dm zsap&BsMGO%V|iNxI-M%gnU3>JeiDiQ(j3kXYNLe%EjGIq|J3|norYvnD^ijuKo%% z(_c7Rd)cBbVL^89Q$3{Y7q6@fs0U&z^{4vW14LA*GXuV{T?m(N?&-Hvx6}J%23Db( zmEDd5q^g{bWRX`XeYb$_NZ~6w?rl2goRNz(S)s^F9tQ3tU7O%#xAK=SYyo{^Uy9mU=$}py-X%|2Lhl!nYtLl)aKku!7Ly;;OKZpCiR`K}NQ#?ML z`h6p8c|^{t`i3S;YWpl zO;gUe@|Uu*Gpc&8Z>KwXzBg{m;!Fv%ao6x>uk4lNhV<1)*FSnqFlXZQE*9!rUmB%E zn<$13{d9ptA8JKNv+TSsVIC@W9S1ZgS&|ct*tcstczZ= z^Y)W;g*fOo#|o8I&2Rd#Q7VlS^h-Z|p@Sg(CFogr@~uJZsMwXFzZIN@-fF91F@Bup znBz0uFS4}mk#$UG?;m#T^i1=Y^^Tqu=YjM}0LX_I@Ca8<_M*1d6npn9Zs1k3sNXb* z7(w0Y=upq^BU2>fjd2KY-MeQ)dD*{!&mE0=4yGL7jcT@wX8{&7ol4D za;AY)sbICdxP(Gn+Uu#orqAvtb_2+13FcsbDJv|g-&3^LCAMRvgFhc0?oLp$KYw+t zVGK}p#XV`;ej6C#kF|SRgCkFCf-<{^6YuPcQ}#`Mxnq{oxncK5OOK7mh2H=N`vT;h zo%CP2Zsr$@H^AdGK00COL^uG#=4TGt^P|RB8}X*PcIm~>LQ(FUAJh+T;&?bo?E3kCn%N^^fvZK2{=RrG))@0ixr6xkE$XZm2$ zwYP=jhi!D4p>zMEBSg7@tirM&;7ZJUO1#6C`z7^m$9<P2%m!G7E-%$962W}7}!pOAx1<8OmV13M1fku9(Mju~dE+>)tlyRir@KlX4(d>4} zzaJ_@&)#A~+aN^!8cXai4pijgOj$g*v+xe8e=ok;V&9|QZWC@4bMo>BLw{h_KM)fo x1>tPlh`-JDJB##{-rRRO5&rpW{5_@aZ>M}!oBR9&4g&J?10Aj50s=z^@;^dP_3Z!v literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..146bc45a14 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,30 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/jobs.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/jobs.yaml new file mode 100644 index 0000000000..6955e3b309 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/jobs.yaml @@ -0,0 +1,102 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-create + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": post-install, post-upgrade, post-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-create + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: create-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/manifest.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/manifest.yaml new file mode 100644 index 0000000000..8dc9dfb447 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/rbac.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/rbac.yaml new file mode 100644 index 0000000000..a4d498b0fa --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.4+up57.0.3/values.yaml b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/values.yaml new file mode 100644 index 0000000000..99e63600c4 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.4+up57.0.3/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/index.yaml b/index.yaml index 39ca79ed74..2c7c4be85f 100755 --- a/index.yaml +++ b/index.yaml @@ -21219,6 +21219,20 @@ entries: urls: - assets/rancher-monitoring-crd/rancher-monitoring-crd-105.0.0+up57.0.3.tgz version: 105.0.0+up57.0.3 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd + apiVersion: v2 + created: "2025-02-25T21:46:36.555096828-03:00" + description: Installs the CRDs for rancher-monitoring. + digest: 009515ed4296f6e01eaffae24012e0f2a6b3350a155c22237530c366c3e02ccd + name: rancher-monitoring-crd + type: application + urls: + - assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.4+up57.0.3.tgz + version: 104.1.4+up57.0.3 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index cdf2216f5d..d4065d7c42 100644 --- a/release.yaml +++ b/release.yaml @@ -12,3 +12,5 @@ fleet-agent: - 104.1.5+up0.10.9 rancher-monitoring: - 104.1.4+up57.0.3 +rancher-monitoring-crd: + - 104.1.4+up57.0.3 From bc746d82f4e9850ef77eba74d4e010ccb761a537 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:47:41 -0300 Subject: [PATCH 10/19] fp: harvester-csi-driver 104.0.5+up0.1.23 --- .../harvester-csi-driver-104.0.5+up0.1.23.tgz | Bin 0 -> 4029 bytes .../104.0.5+up0.1.23/.helmignore | 23 +++ .../104.0.5+up0.1.23/Chart.yaml | 22 +++ .../104.0.5+up0.1.23/questions.yml | 11 ++ .../104.0.5+up0.1.23/templates/NOTES.txt | 1 + .../104.0.5+up0.1.23/templates/_helpers.tpl | 62 +++++++ .../104.0.5+up0.1.23/templates/csidriver.yaml | 10 ++ .../104.0.5+up0.1.23/templates/daemonset.yaml | 152 ++++++++++++++++++ .../templates/deployment.yaml | 120 ++++++++++++++ .../104.0.5+up0.1.23/templates/rbac.yaml | 75 +++++++++ .../templates/storageclass.yaml | 10 ++ .../104.0.5+up0.1.23/values.yaml | 56 +++++++ index.yaml | 26 +++ release.yaml | 2 + 14 files changed, 570 insertions(+) create mode 100644 assets/harvester-csi-driver/harvester-csi-driver-104.0.5+up0.1.23.tgz create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/.helmignore create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/Chart.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/questions.yml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/NOTES.txt create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/_helpers.tpl create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/csidriver.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/daemonset.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/deployment.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/rbac.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/templates/storageclass.yaml create mode 100644 charts/harvester-csi-driver/104.0.5+up0.1.23/values.yaml diff --git a/assets/harvester-csi-driver/harvester-csi-driver-104.0.5+up0.1.23.tgz b/assets/harvester-csi-driver/harvester-csi-driver-104.0.5+up0.1.23.tgz new file mode 100644 index 0000000000000000000000000000000000000000..81171b545ac5ef762eab8eb1ab1fa0c693dc243c GIT binary patch literal 4029 zcmV;u4?^%CiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH+#bK5wQ`OII@r(|n0*(LSxBYUdzRb?&Du4^W?D@tW*>*^{6 zku3>p5a0lyY|SM1+phrdE$ZP%oXL#v!KO%{A3%4b(G6fu4%E?k)#Wh-P1Xd zI#`mJJ$sDO>-Boahllomuh(n-?;rGEKkFa9KI$C~2FLx^&wBl%!O_7p=sgPdEhSZ& z$Y;F=<7zVZjTFMU&_q+g)zAY7i6%_Uf{TtU{VPQ zaR?TQS}KikYgM6!z$i~|TfZhKW2%+ExmLz}o!CL}pBb%#px;|ht3-xGVb(isI@j7P zrG6;lL~!K#F3l39C6c(LwHKIf^+-ZL%h2&INnhltYa9`$ts<;<};2`LE4z8lVZ0?YCXtJg?!2Mv5 z0Zyb?P{Z~8fLEF~DDMXY`-W&u42b(c&)e2O9|VJ-?*Sw!V^@OFa5;qY>4nf&5*2do z+1VhMfFX1`-ZW)w<)!x=-p{FmDMc1RNH|OoQiTy{0V=_ePM1zTgp{Zf!6V9NFxEnn z8J;qt6bNo>%!ServebxU;F(|5S{9peG8pQ2Lg*2dL+CDu>@qs(W*@wei8MMDd`f4g zs}zP3^@W|+I!5{o)GMOrL+BK3^Zt|9IOw7dyP<9GPbovy?RcIMu`x0XwXg$VkV%dj zRo$`iv|uPpN6rVFPBGL&xDexTj#0|c4#G7qox`-CQd`eW63Xo-kr)U<4&m&UDy_DH zvGOkk!=O%9D7cn_`3WN&o1lMA32Q(;yTvfoI{-!mtH5?0{cZwQhXi z0*=O-K=8pWKC9jyd;%>~9>Vc~eWdX?olfa3bo>rvu#iXQ(UE#8k!S=`U~u-$H~==5 zW!4IoiNN~_9c)`)8w`da1rrQOst}Zj5q>DV5XZXpI290yz-Uahb2+tfMqf^x&pEmn zo4Ur~G_L4(fpW_>9`!dO7!&%bupwqGOw` zbl!n$zwgrdOciimSVo2?yN_;3W6>oJNNt`e=&ZfGBXNnOTu-6%C*}W1bsDf7(%ogP z{jny2m7z7dlLfBKu-f6xWX08DWP)thA+}A*@on-|?Q-{=Esf!mS-&weWq1D4fzHRy z-IYtkvEYtewIjK%vZHs7EDqGXYgrf?e}eLmrO_5v14qo3$|_qc{g@#PD$xt~BAHgVU;K zHFo}CmqOXvr_k&y=CNGm&Q<)FRs5J&Y&A%3nU%+{TbEZYa^CIS3xG8S`HxOXkKwlY z{I%!irzl(ezfjRknjR1g-nRZbI5?{Ne}mqjw~zlmNvWoUlnZ$U*MD<%uepCPZ$47hpe~MDof087s)cTPf`Z4Mjt-p)! z0Z_@*uL~Y|kU2^QhxHFOf@g(y_X#-V5poTO225^uR3&5%5EC8Fe=OkJ3{{y|U&%gg z$G_@jmUJ9Dc5)rcD5!LyDXd1!W^x`%(BB+E0D0#)gk?59N8SKan` z#`q_J65(|PG4e%|1t+u_k+p9{F2F!)gXS`vMsD)~-Qe6Bd9Uzg+Xe<`OTIBE(`fMd z^Kj>8n#}U?tWbDWNu{lTD)VghmMot?>;08T9BYvb8y$tWrSy~~6md*=wEDsHdv`*4 zSIt|$`C;p88G|pU(5(v0n!z_h)f}#}*a7Xns(f=&uqkJ2io!6JR4-2j*LbU&T|*a? z;jH4yYn^hN{-DvxiG=HP*~RGU^8Dg`yRy6dj(r{8m3SwW@|?8-&U-w|u$Qeq)(>CR(ZN=)h zRIrWRs2jU&ZGo9FTS=>qOS^qbIlUcryI;=C6$2dGmfw>IpSfo*1sFeYxUa@%#LLc)mK;@2=@03k`pgfglrswQV7YGKxg@xR6&-;Ca# zUp$7Xbj&(i(PAP1B$DWovte=w+`>tOi&g-3%4mM9 zH&x3frIn`5kBtdh2t9oaJ`Ju7c#LGnB1+kc;JW6FPW<-ot%7a$CY_IJ?-a@f4e?DF z8oU?xSolFU$KFsqFH=929ZhAm=02yqsTThtF>7DXv`pYD*ecti%36)RQuoB=%vf#x zw845coSVvKv7p>3R-dHY`?`g!?kcOO+jel(Ojrr7*ppVb-A!$-g3p^Yohi3Xdt}90 z$mOMUDX=;Lb!W@ph<}r1-G*${_U(xD5v=sxX$TyjcFcQ}abs|w!}mhMyC2T_%`rEg{@HC}EKrrvvc@i_UCYC+)zWgNwuPd>7oo_% zBT0#f-f&IZh9!|WMR~7r@!wfJB$?v;tiT}w72_Jk9%$EKChpz`K=Rf;{!+rekDN4g{bfLd-&7Jv{_W%06U%TbX z$}dUvTuAzhbz@dfH@I6r-U)_(r}l=B2Q&OAteAN{bM{F>?Rkj}{CEH>?A(ibl}`SR zq0TN5X_bf4OyAYV-9-`}HiW;Cu}c|dJ(!Q3na80M3LzuPt88?@&F2!8NM(qwqU{)! ze_NoOxc4*Eoxeio#yCIbfl*~&L}qqI^OPnLaZjP0TOEuhA@#F9?KFgnX4OKHkkIZ4 zTT4++NI_YKy+%=zf6T~jA+6I{C8d)TXL>RZrgK*+z5o#+BQ$PO^7RPmDou2{LXShz z0=XdpEqL<5XGE!$E^lYF4G$-YnhPClN0<$6)?g#X$3$dYYvgyux3dnJRobq4)2gfO zka1yYUV(cDMZ5#HL_`nbfh37)hny$ku#FjmU{|$p{d`|3-BD&-I4RyHF4g{Dr-8c( z&e_Qb^YFt5}io{cx-Lj`;iXG2xSN~w;-rhN*%B||v%MRC_J(g}l%wxtQJGpvJk1`Mc zt%z_4=W&b?B^v+0F@{$C59Y{XY9o7(fn%HgA2rYa_m2*b_vingq&$Z!qBTmcKnrkV z4mWe;Fi9!1_hlv|ye2bLL4Mz+N|QuLtw7C@F_TH-gC(ApZRb)hKNe# z1v9F<_OBz~3nstFuKk<8n9sWAPySOac((*JA>nnJ*t@Tk_bN~~iT5g)kZbQ%pkwna zB%OJ${?B_3ABdzPRd7B!Q(llrVV25XKqDkwS6zyKd%;45BEs&sU{+f7|K}8ywc)|? zF>Zj~I{&ZS|9{-??eG78k`fY)Ga>D1uZ{P&ueol&5EavWK_n#;mLGv=92m%hD?>On zU%?%auzF{0g$Gc(e!erVT;XetNq*y8rFqct0A?;Y*2ymYF|S;rL1{_(%mTJwhR|u8 jD(F@ZYkZqmxbDlo?90B~tNece00960K!kzi0B8UJ1?BiW literal 0 HcmV?d00001 diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/.helmignore b/charts/harvester-csi-driver/104.0.5+up0.1.23/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/Chart.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/Chart.yaml new file mode 100644 index 0000000000..ffeff3b5d1 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Harvester CSI Driver + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: kube-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: harvester-csi-driver + catalog.cattle.io/ui-component: harvester-csi-driver + catalog.cattle.io/upstream-version: 0.1.23 +apiVersion: v2 +appVersion: v0.2.2 +description: A Helm chart for Harvester CSI driver +keywords: +- infrastructure +- harvester +maintainers: +- name: harvester +name: harvester-csi-driver +type: application +version: 104.0.5+up0.1.23 diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/questions.yml b/charts/harvester-csi-driver/104.0.5+up0.1.23/questions.yml new file mode 100644 index 0000000000..0c703a0f29 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/questions.yml @@ -0,0 +1,11 @@ +categories: +- infrastructure +- harvester +namespace: kube-system +questions: +- variable: cloudConfig.hostPath + label: Cloud config file path + description: "Specify the path of the cloud config." + group: "Default" + type: string + default: "/etc/kubernetes/cloud-config" diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/NOTES.txt b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/NOTES.txt new file mode 100644 index 0000000000..ba09c6d518 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/NOTES.txt @@ -0,0 +1 @@ +Successfully deployed Harvester CSI driver to the {{ .Release.Namespace }} namespace. diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/_helpers.tpl b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/_helpers.tpl new file mode 100644 index 0000000000..def471b21d --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "harvester-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "harvester-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "harvester-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "harvester-csi-driver.labels" -}} +helm.sh/chart: {{ include "harvester-csi-driver.chart" . }} +{{ include "harvester-csi-driver.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "harvester-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "harvester-csi-driver.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Global system default registry +*/}} +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/csidriver.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/csidriver.yaml new file mode 100644 index 0000000000..5fc6ec84a2 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: driver.harvesterhci.io +spec: + attachRequired: true + fsGroupPolicy: ReadWriteOnceWithFSType + podInfoOnMount: true + volumeLifecycleModes: + - Persistent diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/daemonset.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/daemonset.yaml new file mode 100644 index 0000000000..6ccb3203a0 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/daemonset.yaml @@ -0,0 +1,152 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "harvester-csi-driver.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "harvester-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + component: csi-driver + {{- include "harvester-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + component: csi-driver + {{- include "harvester-csi-driver.selectorLabels" . | nindent 8 }} + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path={{ .Values.kubeletRootDir }}/harvester-plugins/driver.harvesterhci.io/csi.sock + env: + - name: ADDRESS + value: /csi/csi.sock + image: {{ template "system_default_registry" . }}{{ .Values.image.csi.nodeDriverRegistrar.repository }}:{{ .Values.image.csi.nodeDriverRegistrar.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/driver.harvesterhci.io-reg.sock + /csi//* + name: node-driver-registrar + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi/ + name: socket-dir + - mountPath: /registration + name: registration-dir + - args: + - --nodeid=$(NODE_ID) + - --endpoint=$(CSI_ENDPOINT) + - --kubeconfig=/var/lib/harvester/cloud-provider-config + {{- if .Values.hostStorageClass }} + - --host-storage-class={{ .Values.hostStorageClass }} + {{- end }} + env: + - name: NODE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + image: {{ template "system_default_registry" . }}{{ .Values.image.harvester.csiDriver.repository }}:{{ .Values.image.harvester.csiDriver.tag | default .Chart.AppVersion }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -f /csi//* + name: harvester-csi-driver + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - name: cloud-config + readOnly: true + mountPath: /var/lib/harvester + - name: kubernetes + readOnly: true + mountPath: /etc/kubernetes + - mountPath: {{ .Values.kubeletRootDir }}/plugins/kubernetes.io/csi + mountPropagation: Bidirectional + name: kubernetes-csi-dir + - mountPath: /csi/ + name: socket-dir + - mountPath: {{ .Values.kubeletRootDir }}/pods + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /rootfs + mountPropagation: Bidirectional + name: host + - mountPath: /lib/modules + name: lib-modules + readOnly: true + hostPID: true + serviceAccountName: {{ include "harvester-csi-driver.name" . }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: cloud-config + {{- if .Values.cloudConfig.secretName }} + secret: + secretName: {{ .Values.cloudConfig.secretName }} + {{- else }} + hostPath: + path: {{ .Values.cloudConfig.hostPath }} + type: DirectoryOrCreate + {{- end }} + - hostPath: + path: /etc/kubernetes + type: DirectoryOrCreate + name: kubernetes + - hostPath: + path: {{ .Values.kubeletRootDir }}/plugins/kubernetes.io/csi + type: DirectoryOrCreate + name: kubernetes-csi-dir + - hostPath: + path: {{ .Values.kubeletRootDir }}/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: {{ .Values.kubeletRootDir }}/harvester-plugins/driver.harvesterhci.io + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: {{ .Values.kubeletRootDir }}/pods + type: DirectoryOrCreate + name: pods-mount-dir + - hostPath: + path: /dev + name: host-dev + - hostPath: + path: /sys + name: host-sys + - hostPath: + path: / + name: host + - hostPath: + path: /lib/modules + name: lib-modules diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/deployment.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/deployment.yaml new file mode 100644 index 0000000000..0b592251ce --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/deployment.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "harvester-csi-driver.name" . }}-controllers + namespace: {{ .Release.Namespace }} + labels: + {{- include "harvester-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicasCount }} + selector: + matchLabels: + component: csi-controllers + {{- include "harvester-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + component: csi-controllers + {{- include "harvester-csi-driver.selectorLabels" . | nindent 8 }} + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --timeout=1m50s + - --leader-election + - --leader-election-namespace=$(POD_NAMESPACE) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ template "system_default_registry" . }}{{ .Values.image.csi.resizer.repository }}:{{ .Values.image.csi.resizer.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: csi-resizer + volumeMounts: + - mountPath: /csi/ + name: socket-dir + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --timeout=1m50s + - --leader-election + - --leader-election-namespace=$(POD_NAMESPACE) + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ template "system_default_registry" . }}{{ .Values.image.csi.provisioner.repository }}:{{ .Values.image.csi.provisioner.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: csi-provisioner + volumeMounts: + - mountPath: /csi/ + name: socket-dir + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --timeout=1m50s + - --leader-election + - --leader-election-namespace=$(POD_NAMESPACE) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ template "system_default_registry" . }}{{ .Values.image.csi.attacher.repository }}:{{ .Values.image.csi.attacher.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: csi-attacher + volumeMounts: + - mountPath: /csi/ + name: socket-dir + serviceAccountName: {{ include "harvester-csi-driver.name" . }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + # For RKE1 + - matchExpressions: + - key: node-role.kubernetes.io/controlplane + operator: Exists + # For RKE2 + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [harvester-csi-driver] + - key: component + operator: In + values: [csi-controllers] + topologyKey: kubernetes.io/hostname + volumes: + - hostPath: + path: {{ .Values.kubeletRootDir }}/harvester-plugins/driver.harvesterhci.io + type: DirectoryOrCreate + name: socket-dir diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/rbac.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/rbac.yaml new file mode 100644 index 0000000000..2ba042a26b --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/rbac.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "harvester-csi-driver.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "harvester-csi-driver.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "harvester-csi-driver.name" . }} + labels: + {{- include "harvester-csi-driver.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "harvester-csi-driver.name" . }} +subjects: + - kind: ServiceAccount + name: {{ include "harvester-csi-driver.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "harvester-csi-driver.name" . }} + labels: + {{- include "harvester-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csistoragecapacities" ] + verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: [ "get", "list", "watch", "create","update", "patch", "delete" ] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create","update", "patch", "delete" ] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "apps" ] + resources: [ "replicasets" ] + verbs: [ "get" ] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "patch" ] diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/storageclass.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/storageclass.yaml new file mode 100644 index 0000000000..a29c9c7da0 --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/templates/storageclass.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: harvester + annotations: + storageclass.kubernetes.io/is-default-class: "true" +allowVolumeExpansion: true +provisioner: driver.harvesterhci.io +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/charts/harvester-csi-driver/104.0.5+up0.1.23/values.yaml b/charts/harvester-csi-driver/104.0.5+up0.1.23/values.yaml new file mode 100644 index 0000000000..fe6b120cbd --- /dev/null +++ b/charts/harvester-csi-driver/104.0.5+up0.1.23/values.yaml @@ -0,0 +1,56 @@ +# Default values for harvester-csi-driver. + +replicasCount: 3 + +image: + harvester: + csiDriver: + repository: rancher/harvester-csi-driver + # Overrides the image tag whose default is the chart appVersion. + tag: "v0.2.2" + csi: + nodeDriverRegistrar: + repository: rancher/mirrored-longhornio-csi-node-driver-registrar + tag: v2.3.0 + resizer: + repository: rancher/mirrored-longhornio-csi-resizer + tag: v1.2.0 + provisioner: + repository: rancher/mirrored-longhornio-csi-provisioner + tag: v2.1.2 + attacher: + repository: rancher/mirrored-longhornio-csi-attacher + tag: v3.2.1 + pullPolicy: IfNotPresent + +nameOverride: "" +fullnameOverride: "" + +# This field can be used to specify the corresponding StorageClass on the host cluster. +hostStorageClass: "" +kubeletRootDir: /var/lib/kubelet +cloudConfig: + secretName: "" + hostPath: "/var/lib/rancher/rke2/etc/config-files/" + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - effect: NoSchedule + key: kubevirt.io/drain + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + - effect: NoExecute + key: node-role.kubernetes.io/etcd + operator: Equal + - key: cattle.io/os + operator: Equal + value: "linux" + effect: NoSchedule + +global: + cattle: + systemDefaultRegistry: "" diff --git a/index.yaml b/index.yaml index 2c7c4be85f..aca0d3c446 100755 --- a/index.yaml +++ b/index.yaml @@ -4049,6 +4049,32 @@ entries: urls: - assets/harvester-csi-driver/harvester-csi-driver-105.0.0+up0.1.21.tgz version: 105.0.0+up0.1.21 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Harvester CSI Driver + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: kube-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: harvester-csi-driver + catalog.cattle.io/ui-component: harvester-csi-driver + catalog.cattle.io/upstream-version: 0.1.23 + apiVersion: v2 + appVersion: v0.2.2 + created: "2025-02-25T21:47:20.718578748-03:00" + description: A Helm chart for Harvester CSI driver + digest: e1bc2269dbea2acdd61de4c73de6bd60d9e81be6ba53aea744a2ff295b027c84 + keywords: + - infrastructure + - harvester + maintainers: + - name: harvester + name: harvester-csi-driver + type: application + urls: + - assets/harvester-csi-driver/harvester-csi-driver-104.0.5+up0.1.23.tgz + version: 104.0.5+up0.1.23 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/display-name: Harvester CSI Driver diff --git a/release.yaml b/release.yaml index d4065d7c42..5c4897c6b3 100644 --- a/release.yaml +++ b/release.yaml @@ -14,3 +14,5 @@ rancher-monitoring: - 104.1.4+up57.0.3 rancher-monitoring-crd: - 104.1.4+up57.0.3 +harvester-csi-driver: + - 104.0.5+up0.1.23 From e80cb0fcd65f0feb51dd90a61b42b3037cc58298 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:48:35 -0300 Subject: [PATCH 11/19] fp: rancher-cis-benchmark 6.7.0 --- .../rancher-cis-benchmark-6.7.0.tgz | Bin 0 -> 6162 bytes charts/rancher-cis-benchmark/6.7.0/Chart.yaml | 22 ++ charts/rancher-cis-benchmark/6.7.0/README.md | 9 + .../rancher-cis-benchmark/6.7.0/app-readme.md | 34 +++ .../6.7.0/templates/_helpers.tpl | 27 +++ .../6.7.0/templates/alertingrule.yaml | 14 ++ .../6.7.0/templates/benchmark-aks-1.0.yaml | 8 + .../6.7.0/templates/benchmark-cis-1.8.yaml | 9 + .../6.7.0/templates/benchmark-cis-1.9.yaml | 8 + .../6.7.0/templates/benchmark-eks-1.2.0.yaml | 8 + .../6.7.0/templates/benchmark-gke-1.2.0.yaml | 9 + .../6.7.0/templates/benchmark-gke-1.6.0.yaml | 8 + .../benchmark-k3s-cis-1.8-hardened.yaml | 9 + .../benchmark-k3s-cis-1.8-permissive.yaml | 9 + .../templates/benchmark-k3s-cis-1.9.yaml | 8 + .../benchmark-rke-cis-1.8-hardened.yaml | 8 + .../benchmark-rke-cis-1.8-permissive.yaml | 8 + .../benchmark-rke2-cis-1.8-hardened.yaml | 9 + .../benchmark-rke2-cis-1.8-permissive.yaml | 9 + .../templates/benchmark-rke2-cis-1.9.yaml | 8 + .../6.7.0/templates/cis-roles.yaml | 49 ++++ .../6.7.0/templates/configmap.yaml | 18 ++ .../6.7.0/templates/deployment.yaml | 61 +++++ .../templates/network_policy_allow_all.yaml | 15 ++ .../patch_default_serviceaccount.yaml | 29 +++ .../6.7.0/templates/rbac.yaml | 211 ++++++++++++++++++ .../6.7.0/templates/scanprofile-cis-1.8.yaml | 9 + .../6.7.0/templates/scanprofile-cis-1.9.yaml | 9 + .../scanprofile-k3s-cis-1.8-hardened.yml | 9 + .../scanprofile-k3s-cis-1.8-permissive.yml | 9 + .../templates/scanprofile-k3s-cis-1.9.yaml | 9 + .../scanprofile-rke-1.8-hardened.yaml | 9 + .../scanprofile-rke-1.8-permissive.yaml | 9 + .../scanprofile-rke2-cis-1.8-hardened.yml | 9 + .../scanprofile-rke2-cis-1.8-permissive.yml | 9 + .../templates/scanprofile-rke2-cis-1.9.yaml | 9 + .../6.7.0/templates/scanprofileaks.yml | 9 + .../6.7.0/templates/scanprofileeks.yml | 9 + .../6.7.0/templates/scanprofilegke-1.6.0.yml | 9 + .../6.7.0/templates/scanprofilegke.yml | 9 + .../6.7.0/templates/serviceaccount.yaml | 14 ++ .../6.7.0/templates/validate-install-crd.yaml | 17 ++ .../rancher-cis-benchmark/6.7.0/values.yaml | 53 +++++ index.yaml | 26 +++ release.yaml | 2 + 45 files changed, 845 insertions(+) create mode 100644 assets/rancher-cis-benchmark/rancher-cis-benchmark-6.7.0.tgz create mode 100644 charts/rancher-cis-benchmark/6.7.0/Chart.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/README.md create mode 100644 charts/rancher-cis-benchmark/6.7.0/app-readme.md create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/_helpers.tpl create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/alertingrule.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-aks-1.0.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.8.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-eks-1.2.0.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.2.0.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.6.0.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-hardened.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-permissive.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-hardened.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-permissive.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-hardened.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-permissive.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/cis-roles.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/configmap.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/deployment.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/network_policy_allow_all.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/patch_default_serviceaccount.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/rbac.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.8.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-hardened.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-permissive.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-hardened.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-permissive.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-hardened.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-permissive.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.9.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofileaks.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofileeks.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke-1.6.0.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke.yml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/serviceaccount.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/templates/validate-install-crd.yaml create mode 100644 charts/rancher-cis-benchmark/6.7.0/values.yaml diff --git a/assets/rancher-cis-benchmark/rancher-cis-benchmark-6.7.0.tgz b/assets/rancher-cis-benchmark/rancher-cis-benchmark-6.7.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..e462e4ba894191478328ede01c723d7985eea12f GIT binary patch literal 6162 zcmV+t813gDiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKB#Z`(N1|Ni^cs~9-LB$9+tF|q(WBvHVoj)$ZSutQiEMUvTs3WMGLOZYLVptAft8e{tgb3qs#!Lz^q0It(N zae8(S{sOKuaMj0Wm5-__M2I{*0wo9wF-MYdTignX7nP4NF@k{5=#R>~A?6Drg`Ji< zToUTDrKm2=*$wfru%|a%w_tV8@q$Q5%!P-j=-#-M&nJVB6Ta`fie>LRZg0Kp90wQ) z9Cvf&cve0qZ$s5xrg^qxET}9N5!+*nkWozKI`$|e|45pC<60;TS6`2v6Q^hSSa_U- zQeV58;oL9*rf3{s0X(9V(5cpylsgFQMV!c6(0vGu0s^HgoMMUffEXDp;wkPkDMK;p zc0FbCgcG+|YrYUzif+KBOmtNPw8rU*9C0(X=Jg!ksa5W1K$jIwdW zZr5{I5Y8FLzMT|JKjhdQoj85V;#jbVdsvL%^A`)?0RD*~5)lf5Tj1DZiv^}WP$pG> zi4c2aatnSuGU?NRsILkH7~NoyYA}!*43AOa5f%x}+M6-rVLgy?rmDf*F(Zcp!GKVY zD-R&Rn<46%zg%i_s;niZH-s~~z*GuY5;=o_DC@3yr%gEo5qUFJ{aZqbvWyO#Pt@3g z&tQpYPLj-H5jE0E-M9%00=6V{s<{V*sv^k$cO>-vg{pvJ4-0|#?V*;R;|0^wWAGv# z+`^b6-8U17IXqi+^)n}Cs$BC(e+KZvXasJrw}`(l@Inu(+kg3%#3KG4Ve!x>tb+r1 z$9xO{8sosgCuBY#A;?sBqAGELCo500az;ArVV>x!F|wx-@o|?jAKL;4*i+uGYaJX| z>e8jY!njr-#qtdi64(UPO^}GF(UJ@#3luakJXdlMjs^TZtq*jNbo|1KOX;W&KK^D| zS=9W@#!7Z}gE=QYzRH%Jpg<^1C^0~j386&Zn)XctHby}lP|5%s7@6=Ywl^1eN`zD$ z+3D!Jv0Hwp9jLyhI?9v5dJM(|@>bxL?RZ-PF5?tTwv zR74y@IxSKzUSMOP1zKQGh`C+PFvswH_sbXS^JiOmCqglF)chrVvIF)P<=|8S=fX@i zoOdbTyVR$3Dkfg$VfCnThU#;!B2(x(=5Wnp1G0LGOu+eviCm2VO6Db%h9`Y2t2CPWrmdY&hPjc+hB;RDIy);? zc3MY{qxe~M0Z%5_lOyQ7W0!IL>T(*}LFerE2nC(>W1VS+h&`WaL*99=X60E ziR$dX_M?POc@*Gn=Yuu&zc)N7%>VkmlcPav|C^wK-85m76UR&j^|qZ`u~{&x!Z1Ro zj>kA=!tvOmd*fPjLj4hZVE$WE!dhU7d?b;I!piF+A@l8+SA0KSlKcEQogJ3EivwT2 zSZY==(t%tY6YBs_FF?N0QMkDP`K7h)%KQ|jkRkKmn)vUG`XN)HwBlexsKY^zh&Vh< zt;~Opk*UM~YLOSsh3z`MEsQ`N{`f6}`D zhUww>$2zv_Z!rU?$NzB=|HFQ}{@V!c7XPV^thWHTodJAJ^lAxxOXbht_C(h+u>jT-a!a* z^PoKe_4qHY{}0_^yZ^fp+8_Q`I?`Hzua~yL|EBK%YUBSwuNeQI9JlZP8lipR|DJaM zb)auw0yIbu$3LIrJ;ne1VR8T8;JBUtH$%I{f0d4`Hw3(9Dd4y@1be6L@W0#mzdtO# z{~xN5pv8X!v_JemB>rzLz*Eyk`0sBQ0Bk=09~_^w`+pjto#Vf(Blp6;-_HM^l(xkG zMgc%Q{)_MbhJ!Z#Z-(}W{|5#Dtp#{W+6w<0E&$fA{}tc=_1!lAS2MIX{NJ+xSPS}g z1+X!y!@s)DS@5W2P&N4PyQTQQ=N`E&{+pm8{`nYr4vOTAaq_?V$N$d!rKqw67c9VQ zegc5}1)&y?0v!L47KP+*oJC>!%UnBeXyObnN#>qap;b1#R5?Y}-H@{h32^1T%8ML_ zj7xLt2J^8P!N-orF_O4*2%RYO<4-;gu=x}s>CHNaohg>;>(YFMPC$h0{I*ukVv4+i ztnjge!cb@itLr*thPqdY&$w{f?#)^h=i903Zbm0$xfTDWqPol|5MwrmbRx_|DUMB{``OMy!Yk|8Ia|8?8tAEeeadS<41H&QN4m zo6}9wcqDvRDJV0uj1MtS2r*C7B*lqyImp`(Tc9QqBe=7*k@}14^j)WC-a*7I$}BJ&dooS{`dMCBI zg>-q8FFzlsi*WusCVZd$|m_8yMcn$uC!&3f_-f-By|7(P@ zcvtbqDsKBJ9r-n3TPB0fCa{M%Ri+6y9+aYI*9m`;2F83c(kTjZEFa7iAZe!ljQI`m z@Tupqh{|`RPP5|z$P`H0&!|L%Vx9@cDm&beNt_GMa(%uwO`hug&!5v&Px@gelXBNt zrPzcoUq<&9m1qiIzC6pfeuVb*Qf8U&)=T@?JQGOoWJf7y_ij(n(xMk zH0Ap0^lz2Txu+5{0A=p)-@X6wm-oN!L&iA0ah-g*7Ccn6g!dY;UY}oHpT0S}xVpT4 zdvakBwfoS z$mmv zJ9E3(pj?SNDDujQnal6Irfs?WpmoiY`9UqWE4GY~C3`$b?MJ4?8s9qHxNQkIThv^Rmr0VD5}SUlft&VeDi75XN}IIJWg= z{j!cQn%a&0tp4$n@isu)HbCQ8fl4N80q@jQsdj}!tCOL4U8A$rY&D+e#Ak!7U(_g+ zGc(W5`W21xqlF^bNrJ}~3#0(_u@hI2d4mj{yuO*&%vvn<(GJ!Bpa0cCkz@;1;4-Jb zS8R^yzdG@OCST&0fI8~sQ~jRKrk%rDfBTzd0gRD1XOqbrvLJElgfw zJO{4E|DOzt`~Tb%_oUtb(+KToGe>sK9EG8iKc?gUvnH#a&;bKg(}wJMm6fj_k~fiD zwJF|3QrPcc8v~dw;wvkkTgI`W^SwUFB-!y|E|l3+JsC1zAA@42c0KvJ+D8#z(1)(% z%f>!H6swQ)vo?d?^ zxs-msPpWbo$B7`t_WYZTOYGKlO_=G8s&3lhHl`=geBmK#Q?XvztGX}s{#CkIR^hB# zmKB@hFd&}pC>79g76h0-Oof+a<%DlUJJ+jcRzHn0GFsBt^?cPGz4@!eSKaXuUN9=K z84PnNeK)(h7LQ4;oRCQ3Nfaa(Gdue3Z%iba@K8YLw7N}tj8rHegv?J$7geC9lG0LB zbyaaOJtri%qK0_Xa*Fmyu-~&;F?BoI-!zQ>hc2aUWEry9&Y~gfQZfKufn=?p*~o_1 zRuk{QeV{ZYpPRF79xozcou{wFV+oCQY$^{+Dsv4vTVOfEk#7RcVqra2mY;-QY9V+h{ck(rkBN9kLN6?Wx!kzTxPO6*Pmg=qA zPu+vxHf@go%};>Z3jgizzZ;?5c0IJXj}Ye6F@yW|I@IT|9Ln#YR`XYg!Ya9+)pE$vH+P% zpSpj)cG?dAo6P}g@$Z)MKMmUXe>1cn{5LfR$mW}W>Yn^IX%qZ&eY)4C+20=${}=uL zVCc5L|89hKiT`Xqg+xc>n`M9BX8u3ywfWzhp?%@MluxB8 z3(&^?8_|aNugU=b`19WerTousd;W7Xv}64DpK1ZHI`jNf_wLtETjPI&1;G0CzvBFV zc+}>9YlilY|Hc*o*X5pn>K^{OX-oWXv;bJY{#T6u`$xU@_g{_Bp77t)0$?uT{8RVk zw@4e{AI-%>-u9QN4*&g=V*KyAt^aR^c8UK?g3>+1Ue)9}y%zD0rVa3q_l5sKuXO&i z>$doBgm#I4+!+3`#(%%H0FR~(@IRf$DXBNL0`>SW`TwJK|6en-OZ=B|m+#pMqzc(u zfvssX{5R+Sd+qzbMrcp?Zv^?R|9?Ey;(x1*k9n%Y|Ix6x{^$1F^Piic!g(*1zpX7I z{oyA))c+>8Xn%I!hNno-lb7+S$M`;~!~YEmh>s*rQh3@PfAGG)glh2b4vq%J_5c2H zyZ+w@9mHusrxKnAn8Ndbv3V5UlZ~l+<948v$>`95j{5iF1@yrF^2IvPxzztOVG;FV z^aJ?BlbIU=%j9OK7;SVjaYo1Fx7^~%^n}{&S;Rrk_6#ociGk(kIigbx&TAB+F$sty zSU9I2&SU#0G9}3bW=Q-U-zN1vm+VqsoMTuoSpvk_RLxMJE@~C z{u}lS>p$-B_^93g)d(HHh0y?beSQh4<%HJ?W0rNL(3uI}z#LK|fJc;Su%2J8=4X%` zumbudRbUhltfHs%96v;Tm}06}D)2ADfT)NH;$v1p&X5ETQ5a+J*^&kf`Pg?X>j2J; zEijqCtxummSu-3gQfpGF&fs*xHu=I$u%-hT7}$|=@$gZxdj1+yb!oyl zoU2b1OSp^~yja-`@!Rv6l%W`PyUXR$@kmmM!}zqjQpuGW>Ar9*>wg%Bg$WKOj4!l3 zQ1)iVmLM5K0_%oNFp?3+hrsa!b9IMB96tKY%hVfVV1X!_V(zG$PJGSr*rGdQOItE~ z8WA6hF7j`bvGVOmV4hq|lbMUi>;F|O=3a7ma3nWdvj|=)ae2zXPWer546POb8`nc4%}cD?s8MRAnUx zvDQD>aH-kaaH=Z+qMldrt9i zY9hrUEN8@RS)Zxl)_krh@%CTn2UJW9@D?8rk8qH zb(!+CU}MbOQGPbr9>$hv_k6$Pk1xzM*%+6DkRi&%N%KF zkd=kH6X)hV*9HQXh)UIe#=#^{Nv45~19F4a98z;BRGh#_9BM`s8;!tJL>!xr{J0YC zZJLHvH!~)ZX-+Kf0K9SBYG06hE!*9d%#i`ick$dAeAp3;^|w?Y+Pr zR`Lja>4HTPmtD{n_2(3-d!TQ|p3FA)Ug-IEQqJ^~9wZjYxXY7aIrV*__@E%T&3CN8 z5o~)^}U0S z)jx2&H%3fk(qXBQCVpShz3Dl_Ziu`&nqnZZE37U<89k|p_})?Oq@nBOyK z50j+7UXpb4`7R|$nQo|Vrm=PbDO*UhFD0wuzfQ)CFd5WrCComwtjvA=U_C&T`9Q3B z^kXK+$<74G@Z8j->9!ViYztxbqGQ$U?=fVFELp?6jVwD?u`1^G8nRkR@O-T#>2}&j zc*vB;wm!1Dg?SghUWE7;0gn(M=I|I7U)8>x0MqquG2PPi2oX{q+mAuaqPM*W$2$<= kcn2c9`g2Z%r(S4nsil@$`pW440ssL2|1W;gcmU`C0KvnUaR2}S literal 0 HcmV?d00001 diff --git a/charts/rancher-cis-benchmark/6.7.0/Chart.yaml b/charts/rancher-cis-benchmark/6.7.0/Chart.yaml new file mode 100644 index 0000000000..49ba52d407 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + catalog.cattle.io/auto-install: rancher-cis-benchmark-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: CIS Benchmark + catalog.cattle.io/kube-version: '>= 1.27.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cis-operator-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: cis.cattle.io.clusterscans/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-cis-benchmark + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: rancher-cis-benchmark +apiVersion: v1 +appVersion: v6.7.0 +description: The cis-operator enables running CIS benchmark security scans on a kubernetes + cluster +icon: https://charts.rancher.io/assets/logos/cis-kube-bench.svg +keywords: +- security +name: rancher-cis-benchmark +version: 6.7.0 diff --git a/charts/rancher-cis-benchmark/6.7.0/README.md b/charts/rancher-cis-benchmark/6.7.0/README.md new file mode 100644 index 0000000000..50beab58ba --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/README.md @@ -0,0 +1,9 @@ +# Rancher CIS Benchmark Chart + +The cis-operator enables running CIS benchmark security scans on a kubernetes cluster and generate compliance reports that can be downloaded. + +# Installation + +``` +helm install rancher-cis-benchmark ./ --create-namespace -n cis-operator-system +``` diff --git a/charts/rancher-cis-benchmark/6.7.0/app-readme.md b/charts/rancher-cis-benchmark/6.7.0/app-readme.md new file mode 100644 index 0000000000..06e5629018 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/app-readme.md @@ -0,0 +1,34 @@ +# Rancher CIS Benchmarks + +This chart enables security scanning of the cluster using [CIS (Center for Internet Security) benchmarks](https://www.cisecurity.org/benchmark/kubernetes/). + +For more information on how to use the feature, refer to our [docs](https://ranchermanager.docs.rancher.com/how-to-guides/advanced-user-guides/cis-scan-guides). + +This chart installs the following components: + +- [cis-operator](https://github.com/rancher/cis-operator) - The cis-operator handles launching the [kube-bench](https://github.com/aquasecurity/kube-bench) tool that runs a suite of CIS tests on the nodes of your Kubernetes cluster. After scans finish, the cis-operator generates a compliance report that can be downloaded. +- Scans - A scan is a CRD (`ClusterScan`) that defines when to trigger CIS scans on the cluster based on the defined profile. A report is created after the scan is completed. +- Profiles - A profile is a CRD (`ClusterScanProfile`) that defines the configuration for the CIS scan, which is the benchmark versions to use and any specific tests to skip in that benchmark. This chart installs a few default `ClusterScanProfile` custom resources with no skipped tests, which can immediately be used to launch CIS scans. +- Benchmark Versions - A benchmark version is a CRD (`ClusterScanBenchmark`) that defines the CIS benchmark version to run using kube-bench as well as the valid configuration parameters for that benchmark. This chart installs a few default `ClusterScanBenchmark` custom resources. +- Alerting Resources - Rancher's CIS Benchmark application lets you run a cluster scan on a schedule, and send alerts when scans finish. + - If you want to enable alerts to be delivered when a cluster scan completes, you need to ensure that [Rancher's Monitoring and Alerting](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/) application is pre-installed and the [Receivers and Routes](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/configuration/#alertmanager-config) are configured to send out alerts. + - Additionally, you need to set `alerts: true` in the Values YAML while installing or upgrading this chart. + +## CIS Kubernetes Benchmark support + +| Source | Kubernetes distribution | scan profile | Kubernetes versions | +|--------|-------------------------|--------------------------------------------------------------------------------------------------------------------|---------------------| +| CIS | any | [cis-1.9](https://github.com/aquasecurity/kube-bench/tree/main/cfg/cis-1.9) | v1.27+ | +| CIS | any | [cis-1.8](https://github.com/aquasecurity/kube-bench/tree/main/cfg/cis-1.8) | v1.26 | +| CIS | rke | [rke-cis-1.8-permissive](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/rke-cis-1.8-permissive) | rke1-v1.26+ | +| CIS | rke | [rke-cis-1.8-hardened](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/rke-cis-1.8-hardened) | rke1-v1.26+ | +| CIS | rke2 | [rke2-cis-1.9](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/rke2-cis-1.9) | rke2-v1.27+ | +| CIS | rke2 | [rke2-cis-1.8-permissive](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/rke2-cis-1.8-permissive) | rke2-v1.26 | +| CIS | rke2 | [rke2-cis-1.8-hardened](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/rke2-cis-1.8-hardened) | rke2-v1.26 | +| CIS | k3s | [k3s-cis-1.9](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/k3s-cis-1.9) | k3s-v1.27+ | +| CIS | k3s | [k3s-cis-1.8-permissive](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/k3s-cis-1.8-permissive) | k3s-v1.26 | +| CIS | k3s | [k3s-cis-1.8-hardened](https://github.com/rancher/security-scan/tree/release/v0.4/package/cfg/k3s-cis-1.8-hardened) | k3s-v1.26 | +| CIS | eks | [eks-1.2.0](https://github.com/aquasecurity/kube-bench/tree/main/cfg/eks-1.2.0) | eks | +| CIS | aks | [aks-1.0](https://github.com/aquasecurity/kube-bench/tree/main/cfg/aks-1.0) | aks | +| CIS | gke | [gke-1.2.0](https://github.com/aquasecurity/kube-bench/tree/main/cfg/gke-1.2.0) | gke-1.20 | +| CIS | gke | [gke-1.6.0](https://github.com/aquasecurity/kube-bench/tree/main/cfg/gke-1.6.0) | gke-1.29+ | diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/_helpers.tpl b/charts/rancher-cis-benchmark/6.7.0/templates/_helpers.tpl new file mode 100644 index 0000000000..b7bb000422 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/_helpers.tpl @@ -0,0 +1,27 @@ +{{/* Ensure namespace is set the same everywhere */}} +{{- define "cis.namespace" -}} + {{- .Release.Namespace | default "cis-operator-system" -}} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/alertingrule.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/alertingrule.yaml new file mode 100644 index 0000000000..1787c88a07 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/alertingrule.yaml @@ -0,0 +1,14 @@ +{{- if .Values.alerts.enabled -}} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: rancher-cis-pod-monitor + namespace: {{ template "cis.namespace" . }} +spec: + selector: + matchLabels: + cis.cattle.io/operator: cis-operator + podMetricsEndpoints: + - port: cismetrics +{{- end }} diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-aks-1.0.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-aks-1.0.yaml new file mode 100644 index 0000000000..1ac866253f --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-aks-1.0.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: aks-1.0 +spec: + clusterProvider: aks + minKubernetesVersion: "1.15.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.8.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.8.yaml new file mode 100644 index 0000000000..e1bbc72dd3 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.8.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: cis-1.8 +spec: + clusterProvider: "" + minKubernetesVersion: "1.26.0" + maxKubernetesVersion: "1.26.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.9.yaml new file mode 100644 index 0000000000..480aad292a --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-cis-1.9.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: cis-1.9 +spec: + clusterProvider: "" + minKubernetesVersion: "1.27.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-eks-1.2.0.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-eks-1.2.0.yaml new file mode 100644 index 0000000000..c1bdd9ed5e --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-eks-1.2.0.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: eks-1.2.0 +spec: + clusterProvider: eks + minKubernetesVersion: "1.15.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.2.0.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.2.0.yaml new file mode 100644 index 0000000000..426f7ec6a8 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.2.0.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: gke-1.2.0 +spec: + clusterProvider: gke + minKubernetesVersion: "1.15.0" + maxKubernetesVersion: "1.28.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.6.0.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.6.0.yaml new file mode 100644 index 0000000000..0538240e53 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-gke-1.6.0.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: gke-1.6.0 +spec: + clusterProvider: gke + minKubernetesVersion: "1.29.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-hardened.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-hardened.yaml new file mode 100644 index 0000000000..db52b9baeb --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-hardened.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: k3s-cis-1.8-hardened +spec: + clusterProvider: k3s + minKubernetesVersion: "1.26.0" + maxKubernetesVersion: "1.26.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-permissive.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-permissive.yaml new file mode 100644 index 0000000000..0afe653586 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.8-permissive.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: k3s-cis-1.8-permissive +spec: + clusterProvider: k3s + minKubernetesVersion: "1.26.0" + maxKubernetesVersion: "1.26.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.9.yaml new file mode 100644 index 0000000000..7b6ef228c8 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-k3s-cis-1.9.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: k3s-cis-1.9 +spec: + clusterProvider: k3s + minKubernetesVersion: "1.27.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-hardened.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-hardened.yaml new file mode 100644 index 0000000000..d3d357c023 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-hardened.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: rke-cis-1.8-hardened +spec: + clusterProvider: rke + minKubernetesVersion: "1.26.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-permissive.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-permissive.yaml new file mode 100644 index 0000000000..208eb777cd --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke-cis-1.8-permissive.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: rke-cis-1.8-permissive +spec: + clusterProvider: rke + minKubernetesVersion: "1.26.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-hardened.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-hardened.yaml new file mode 100644 index 0000000000..1bbb5404f4 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-hardened.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: rke2-cis-1.8-hardened +spec: + clusterProvider: rke2 + minKubernetesVersion: "1.26.0" + maxKubernetesVersion: "1.26.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-permissive.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-permissive.yaml new file mode 100644 index 0000000000..3947056649 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.8-permissive.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: rke2-cis-1.8-permissive +spec: + clusterProvider: rke2 + minKubernetesVersion: "1.26.0" + maxKubernetesVersion: "1.26.x" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.9.yaml new file mode 100644 index 0000000000..57ce01b449 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/benchmark-rke2-cis-1.9.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + name: rke2-cis-1.9 +spec: + clusterProvider: rke2 + minKubernetesVersion: "1.27.0" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/cis-roles.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/cis-roles.yaml new file mode 100644 index 0000000000..23c93dc659 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/cis-roles.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cis-admin +rules: + - apiGroups: + - cis.cattle.io + resources: + - clusterscanbenchmarks + - clusterscanprofiles + - clusterscans + - clusterscanreports + verbs: ["create", "update", "delete", "patch","get", "watch", "list"] + - apiGroups: + - catalog.cattle.io + resources: ["apps"] + resourceNames: ["rancher-cis-benchmark"] + verbs: ["get", "watch", "list"] + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cis-view +rules: + - apiGroups: + - cis.cattle.io + resources: + - clusterscanbenchmarks + - clusterscanprofiles + - clusterscans + - clusterscanreports + verbs: ["get", "watch", "list"] + - apiGroups: + - catalog.cattle.io + resources: ["apps"] + resourceNames: ["rancher-cis-benchmark"] + verbs: ["get", "watch", "list"] + - apiGroups: + - "" + resources: + - configmaps + verbs: ["get", "watch", "list"] diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/configmap.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/configmap.yaml new file mode 100644 index 0000000000..ab91549809 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/configmap.yaml @@ -0,0 +1,18 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: default-clusterscanprofiles + namespace: {{ template "cis.namespace" . }} +data: + # Default ClusterScanProfiles per cluster provider type + rke: |- + <1.21.0: rke-profile-permissive-1.20 + >=1.21.0: rke-profile-permissive-1.8 + rke2: |- + <1.21.0: rke2-cis-1.20-profile-permissive + >=1.21.0: rke2-cis-1.9-profile + eks: "eks-profile" + gke: "gke-profile-1.6.0" + aks: "aks-profile" + k3s: "k3s-cis-1.9-profile" + default: "cis-1.9-profile" diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/deployment.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/deployment.yaml new file mode 100644 index 0000000000..8c9f72f5de --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cis-operator + namespace: {{ template "cis.namespace" . }} + labels: + cis.cattle.io/operator: cis-operator +spec: + selector: + matchLabels: + cis.cattle.io/operator: cis-operator + template: + metadata: + labels: + cis.cattle.io/operator: cis-operator + spec: + serviceAccountName: cis-operator-serviceaccount + containers: + - name: cis-operator + image: '{{ template "system_default_registry" . }}{{ .Values.image.cisoperator.repository }}:{{ .Values.image.cisoperator.tag }}' + imagePullPolicy: IfNotPresent + ports: + - name: cismetrics + containerPort: {{ .Values.alerts.metricsPort }} + env: + - name: SECURITY_SCAN_IMAGE + value: {{ template "system_default_registry" . }}{{ .Values.image.securityScan.repository }} + - name: SECURITY_SCAN_IMAGE_TAG + value: {{ .Values.image.securityScan.tag }} + - name: SONOBUOY_IMAGE + value: {{ template "system_default_registry" . }}{{ .Values.image.sonobuoy.repository }} + - name: SONOBUOY_IMAGE_TAG + value: {{ .Values.image.sonobuoy.tag }} + - name: CIS_ALERTS_METRICS_PORT + value: '{{ .Values.alerts.metricsPort }}' + - name: CIS_ALERTS_SEVERITY + value: {{ .Values.alerts.severity }} + - name: CIS_ALERTS_ENABLED + value: {{ .Values.alerts.enabled | default "false" | quote }} + - name: CLUSTER_NAME + value: '{{ .Values.global.cattle.clusterName }}' + - name: CIS_OPERATOR_DEBUG + value: '{{ .Values.image.cisoperator.debug }}' + {{- if .Values.securityScanJob.overrideTolerations }} + - name: SECURITY_SCAN_JOB_TOLERATIONS + value: '{{ .Values.securityScanJob.tolerations | toJson }}' + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/network_policy_allow_all.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/network_policy_allow_all.yaml new file mode 100644 index 0000000000..6ed5d645ea --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/network_policy_allow_all.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ template "cis.namespace" . }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/patch_default_serviceaccount.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/patch_default_serviceaccount.yaml new file mode 100644 index 0000000000..e78a6bd08a --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/patch_default_serviceaccount.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + spec: + serviceAccountName: cis-operator-serviceaccount + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + restartPolicy: Never + containers: + - name: sa + image: "{{ template "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", {{ template "cis.namespace" . }}] + + backoffLimit: 1 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/rbac.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/rbac.yaml new file mode 100644 index 0000000000..cba20d18c0 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/rbac.yaml @@ -0,0 +1,211 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + name: cis-operator-clusterrole +rules: +- apiGroups: + - "cis.cattle.io" + resources: + - "*" + verbs: + - "*" +- apiGroups: + - "" + resources: + - "pods" + - "services" + - "configmaps" + - "nodes" + - "serviceaccounts" + verbs: + - "get" + - "list" + - "create" + - "update" + - "watch" + - "patch" +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - "rolebindings" + - "clusterrolebindings" + - "clusterroles" + - "roles" + verbs: + - "get" + - "list" +- apiGroups: + - "batch" + resources: + - "jobs" + verbs: + - "list" + - "create" + - "patch" + - "update" + - "watch" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + name: cis-scan-ns +rules: +- apiGroups: + - "" + resources: + - "namespaces" + - "nodes" + - "pods" + - "serviceaccounts" + - "services" + - "replicationcontrollers" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - "rolebindings" + - "clusterrolebindings" + - "clusterroles" + - "roles" + verbs: + - "get" + - "list" +- apiGroups: + - "batch" + resources: + - "jobs" + - "cronjobs" + verbs: + - "list" +- apiGroups: + - "apps" + resources: + - "daemonsets" + - "deployments" + - "replicasets" + - "statefulsets" + verbs: + - "list" +- apiGroups: + - "autoscaling" + resources: + - "horizontalpodautoscalers" + verbs: + - "list" +- apiGroups: + - "networking.k8s.io" + resources: + - "networkpolicies" + verbs: + - "get" + - "list" + - "watch" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cis-operator-role + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + namespace: {{ template "cis.namespace" . }} +rules: +- apiGroups: + - "" + resources: + - "services" + verbs: + - "watch" + - "list" + - "get" + - "patch" +- apiGroups: + - "batch" + resources: + - "jobs" + verbs: + - "watch" + - "list" + - "get" + - "delete" +- apiGroups: + - "" + resources: + - "configmaps" + - "pods" + - "secrets" + verbs: + - "*" +- apiGroups: + - "apps" + resources: + - "daemonsets" + verbs: + - "*" +- apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + name: cis-operator-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cis-operator-clusterrole +subjects: +- kind: ServiceAccount + name: cis-operator-serviceaccount + namespace: {{ template "cis.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cis-scan-ns + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cis-scan-ns +subjects: +- kind: ServiceAccount + name: cis-serviceaccount + namespace: {{ template "cis.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + name: cis-operator-rolebinding + namespace: {{ template "cis.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cis-operator-role +subjects: +- kind: ServiceAccount + name: cis-serviceaccount + namespace: {{ template "cis.namespace" . }} +- kind: ServiceAccount + name: cis-operator-serviceaccount + namespace: {{ template "cis.namespace" . }} diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.8.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.8.yaml new file mode 100644 index 0000000000..40be06c946 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.8.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: cis-1.8-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: cis-1.8 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.9.yaml new file mode 100644 index 0000000000..9f0c9f58af --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-cis-1.9.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: cis-1.9-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: cis-1.9 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-hardened.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-hardened.yml new file mode 100644 index 0000000000..03f6695689 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-hardened.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: k3s-cis-1.8-profile-hardened + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: k3s-cis-1.8-hardened diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-permissive.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-permissive.yml new file mode 100644 index 0000000000..39932a4e5b --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.8-permissive.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: k3s-cis-1.8-profile-permissive + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: k3s-cis-1.8-permissive diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.9.yaml new file mode 100644 index 0000000000..3d9ea843b0 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-k3s-cis-1.9.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: k3s-cis-1.9-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: k3s-cis-1.9 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-hardened.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-hardened.yaml new file mode 100644 index 0000000000..54aa08691e --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-hardened.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: rke-profile-hardened-1.8 + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: rke-cis-1.8-hardened diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-permissive.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-permissive.yaml new file mode 100644 index 0000000000..f7d4fdd229 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke-1.8-permissive.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: rke-profile-permissive-1.8 + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: rke-cis-1.8-permissive diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-hardened.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-hardened.yml new file mode 100644 index 0000000000..d0a1180f56 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-hardened.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: rke2-cis-1.8-profile-hardened + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: rke2-cis-1.8-hardened diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-permissive.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-permissive.yml new file mode 100644 index 0000000000..0aa72407c0 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.8-permissive.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: rke2-cis-1.8-profile-permissive + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: rke2-cis-1.8-permissive diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.9.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.9.yaml new file mode 100644 index 0000000000..047d5e86ed --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofile-rke2-cis-1.9.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: rke2-cis-1.9-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: rke2-cis-1.9 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileaks.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileaks.yml new file mode 100644 index 0000000000..ac9f47a8fb --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileaks.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: aks-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: aks-1.0 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileeks.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileeks.yml new file mode 100644 index 0000000000..7cf7936cbf --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofileeks.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: eks-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: eks-1.2.0 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke-1.6.0.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke-1.6.0.yml new file mode 100644 index 0000000000..1fc299fc57 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke-1.6.0.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: gke-profile-1.6.0 + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: gke-1.6.0 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke.yml b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke.yml new file mode 100644 index 0000000000..42fa4f23a2 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/scanprofilegke.yml @@ -0,0 +1,9 @@ +--- +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + name: gke-profile + annotations: + clusterscanprofile.cis.cattle.io/builtin: "true" +spec: + benchmarkVersion: gke-1.2.0 diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/serviceaccount.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ec48ec6224 --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ template "cis.namespace" . }} + name: cis-operator-serviceaccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ template "cis.namespace" . }} + labels: + app.kubernetes.io/name: rancher-cis-benchmark + app.kubernetes.io/instance: release-name + name: cis-serviceaccount diff --git a/charts/rancher-cis-benchmark/6.7.0/templates/validate-install-crd.yaml b/charts/rancher-cis-benchmark/6.7.0/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..562295791b --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/templates/validate-install-crd.yaml @@ -0,0 +1,17 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "cis.cattle.io/v1/ClusterScan" false -}} +# {{- set $found "cis.cattle.io/v1/ClusterScanBenchmark" false -}} +# {{- set $found "cis.cattle.io/v1/ClusterScanProfile" false -}} +# {{- set $found "cis.cattle.io/v1/ClusterScanReport" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-cis-benchmark/6.7.0/values.yaml b/charts/rancher-cis-benchmark/6.7.0/values.yaml new file mode 100644 index 0000000000..3b9f8eebdf --- /dev/null +++ b/charts/rancher-cis-benchmark/6.7.0/values.yaml @@ -0,0 +1,53 @@ +# Default values for rancher-cis-benchmark. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + cisoperator: + repository: rancher/cis-operator + tag: v1.2.5 + securityScan: + repository: rancher/security-scan + tag: v0.4.3 + sonobuoy: + repository: rancher/mirrored-sonobuoy-sonobuoy + tag: v0.57.2 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +securityScanJob: + overrideTolerations: false + tolerations: [] + +affinity: {} + +global: + cattle: + systemDefaultRegistry: "" + clusterName: "" + kubectl: + repository: rancher/kubectl + tag: v1.29.11 + +alerts: + enabled: false + severity: warning + metricsPort: 8080 diff --git a/index.yaml b/index.yaml index aca0d3c446..e19caca2d0 100755 --- a/index.yaml +++ b/index.yaml @@ -12085,6 +12085,32 @@ entries: urls: - assets/rancher-cis-benchmark/rancher-cis-benchmark-105.0.0+up7.0.0.tgz version: 105.0.0+up7.0.0 + - annotations: + catalog.cattle.io/auto-install: rancher-cis-benchmark-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: CIS Benchmark + catalog.cattle.io/kube-version: '>= 1.27.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cis-operator-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: cis.cattle.io.clusterscans/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-cis-benchmark + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: rancher-cis-benchmark + apiVersion: v1 + appVersion: v6.7.0 + created: "2025-02-25T21:48:00.990070872-03:00" + description: The cis-operator enables running CIS benchmark security scans on + a kubernetes cluster + digest: 16e0bd6481cec7934f23be0837985c028e0355f7248c0b61bfeddd7567442973 + icon: https://charts.rancher.io/assets/logos/cis-kube-bench.svg + keywords: + - security + name: rancher-cis-benchmark + urls: + - assets/rancher-cis-benchmark/rancher-cis-benchmark-6.7.0.tgz + version: 6.7.0 - annotations: catalog.cattle.io/auto-install: rancher-cis-benchmark-crd=match catalog.cattle.io/certified: rancher diff --git a/release.yaml b/release.yaml index 5c4897c6b3..0fdd710566 100644 --- a/release.yaml +++ b/release.yaml @@ -16,3 +16,5 @@ rancher-monitoring-crd: - 104.1.4+up57.0.3 harvester-csi-driver: - 104.0.5+up0.1.23 +rancher-cis-benchmark: + - 6.7.0 From 6cdb965c3c22b8b230e75a1c776d036f1f4bab6c Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:49:26 -0300 Subject: [PATCH 12/19] fp: rancher-cis-benchmark-crd 6.7.0 --- .../rancher-cis-benchmark-crd-6.7.0.tgz | Bin 0 -> 1475 bytes .../6.7.0/Chart.yaml | 10 ++ .../rancher-cis-benchmark-crd/6.7.0/README.md | 2 + .../6.7.0/templates/clusterscan.yaml | 149 ++++++++++++++++++ .../6.7.0/templates/clusterscanbenchmark.yaml | 55 +++++++ .../6.7.0/templates/clusterscanprofile.yaml | 37 +++++ .../6.7.0/templates/clusterscanreport.yaml | 40 +++++ index.yaml | 14 ++ release.yaml | 2 + 9 files changed, 309 insertions(+) create mode 100644 assets/rancher-cis-benchmark-crd/rancher-cis-benchmark-crd-6.7.0.tgz create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/Chart.yaml create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/README.md create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/templates/clusterscan.yaml create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/templates/clusterscanbenchmark.yaml create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/templates/clusterscanprofile.yaml create mode 100644 charts/rancher-cis-benchmark-crd/6.7.0/templates/clusterscanreport.yaml diff --git a/assets/rancher-cis-benchmark-crd/rancher-cis-benchmark-crd-6.7.0.tgz b/assets/rancher-cis-benchmark-crd/rancher-cis-benchmark-crd-6.7.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f0bbe0bd2dd214ac8515d21af15f82b2e7d2fb67 GIT binary patch literal 1475 zcmV;!1w8s6iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI@PbK1BQ&NF|7roU%g8wmLgZz=8F^zNEwNUrz1$Qw{&OBzYe zmASe9{YJI{8)U#>XfyS`Flgm(W&7Jt>$MedI!D!|On0Zq={yGY+@&gfJD!6w-U8z2 z>O}K>-yaSJ>96nmo4@{b&%X)=H`hbI7X%5a$VcRjQ3<|rPT>%vmOybt zEZc-c1xBc@UTA}{^ALp*g2t{rbQOP_s~wQ+3o6Y7ACZ@!6QZVyNt6En$hCne(!|V> zj3@6kc@oNv<#qN2G7E_#0#ZhdrdO@=vWz?QZaly9z3*pTtImIev5df=eoLc78&sMC zKS~dF=YQb$2K{pW_k#Yl-{$`%te80<8GbX!?LX~3|6kh_d4iMgt)ju=vw-McF>X+eMU=!mUqhFC)q>xH znU9F44VXlG5oj|>xLwh`5>G6`JBTrzzC8cJqVp-aJChG>6yvkX06K|dPz%op15pY0 zu`p5#N`lrUr2F+iEubf0H9;gMRpP4KHJ*)?60sRe%mw^6S57YwU!rW%MK8?z^ z^G+DDlnf6>yy*TQ#oY+W~p9H>I<9f65RHo~1I)dw?gFQlt)XSsX zEAhg@m76arO_=Hrt?AqYv0SuEeF{WUmMt4CbPCqxg=*YYxAyvRXT%)vW(y z_T*acHSnJLFSs5KHtN6LP5-8?|1M$YuK#kvG+6x=->5ojzh%Jc%1^EU8m{|R`Qxge z-npGWNKLW1^hV#sJu~@;+SF;WP>R}L1@6=Tn_#%9|HB|? z_5TufZvD>))1drMQot(p&%;%VzgiyNu=bznOOJL-x%iuoj3K{0e7^e`9^$I?fAZn> z{ihFJ9G Date: Tue, 25 Feb 2025 21:50:29 -0300 Subject: [PATCH 13/19] fp: fleet 103.1.13+up0.9.15 --- assets/fleet/fleet-103.1.13+up0.9.15.tgz | Bin 0 -> 5344 bytes charts/fleet/103.1.13+up0.9.15/Chart.yaml | 22 ++++ charts/fleet/103.1.13+up0.9.15/README.md | 30 +++++ .../charts/gitjob/.helmignore | 23 ++++ .../charts/gitjob/Chart.yaml | 5 + .../charts/gitjob/templates/_helpers.tpl | 7 ++ .../charts/gitjob/templates/clusterrole.yaml | 38 ++++++ .../gitjob/templates/clusterrolebinding.yaml | 12 ++ .../charts/gitjob/templates/deployment.yaml | 52 ++++++++ .../charts/gitjob/templates/leases.yaml | 23 ++++ .../charts/gitjob/templates/service.yaml | 12 ++ .../gitjob/templates/serviceaccount.yaml | 4 + .../charts/gitjob/values.yaml | 27 +++++ .../103.1.13+up0.9.15/templates/_helpers.tpl | 22 ++++ .../templates/configmap.yaml | 26 ++++ .../templates/deployment.yaml | 102 ++++++++++++++++ .../job_cleanup_clusterregistrations.yaml | 40 ++++++ .../103.1.13+up0.9.15/templates/rbac.yaml | 114 ++++++++++++++++++ .../templates/serviceaccount.yaml | 12 ++ charts/fleet/103.1.13+up0.9.15/values.yaml | 87 +++++++++++++ index.yaml | 26 ++++ release.yaml | 1 + 22 files changed, 685 insertions(+) create mode 100644 assets/fleet/fleet-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml create mode 100644 charts/fleet/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet/fleet-103.1.13+up0.9.15.tgz b/assets/fleet/fleet-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..889602d22a00662ddcdd2c73a51e42c7518a6172 GIT binary patch literal 5344 zcmV<66d&s!iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBxbK5wQ`25XJ(Lecjm6@DHJ^Y$w&CZ?IcCy)Z5})nN-qzNZ z2O?V%Vi4c}pd5|Y_t~%TphQxX<=BpLvnxzh#>Au102hQ zst@irk_hKQ6HNtIQ2-DVO_&&kG0~b~NJWR_TC^!wnlKiDC6{(Ali#OA$K%EpF-lE` z6q5)f;qe${V>SGgp`wJLi2}(r;kKo0RzlUDf-wC_z zF8pBr{GIvpMPpM=QdAj-6B2)y7^88a{X@)YTTD51jwr?m;#EojM zEyaW;sM@26j38#Y(kNAEaVS-r^a#twq}K?;&vJDF`r-F}{QdBr;Y(yl6t>MY_~i{q zMt>?mP5J@Jve{=h{9V}F3lhwbC&*)pDhk>V3!YGIYmBHCnF^7U0mCE!aF9n-e-wiN zKw>5o)k01q7*d8&rxSK!!+%uAS4vRDl4cIxD}xfeC7g^x z%TawP0kDGq@AVh?|K8q)|F0#{l#DQ{?MuIX+;#~7G#N!O@mm7GwmHigI}?n?(+J)S z-wAytQ6bksKt{;*o0V|ew%%t$--1^{7K{=kj;PYo3NX-Pge)I08pDV4QxHR&^XGhk zlA}fihi7j~_vFaf*A3Z#w>KO4O_Ltr@6fN7?uKs8VWM5#Q*1oW7yI;6|f3sXhW5yu1u zQy_4IjKMV*Hyn-*!})NI4vWzoqc-A;qk@|;>M=@-q+KddW0A82v^2;a9l{{z2}1=# zDN-EhHL<+=vQ}v8|*9kYEc^%iE12P)uY9moi779Da6y+7`gi zkctGiVMv(rrCQQhyZq1*-sS1VTajP{t#VU4xI;*Czc{8LAHh&a7>gSaLyg>xQWe8A zgex@=7$cE-fJD0vj>Z^YQ~ri)loPAE^!8E<|1W;VrXq(M!nGYBW8!8;AbBQujYd=Qg(tvk~2%my}hC z9FLKaX#~A~SDEpe?fEzLcgUppH1!)M*m9q4^G2P{54l2Vg#3^H`9nsOx)Cz@G5R5w z?8oQ_v#>uJP_al;0tz!CiN*vNRd&u2krK)k2njR;k|Yw9LgfZ_Lm9I}Fypej-1Ho9 zapo}S^$)_X`LFZ+cCS1C^PnI0_U%Wnzuo(t`TE{{?r(=`5^skE`7;p{76bz!v|07c z@KLW&MP^4~0I&s@V}zOZT0(7rV3hXQED73{>!vFX#4 zB;tlEEs>-%b>70H+=5kOp+heh{U8*^61D#rSRolfN=(xLAR~IL9Dfc*ObiI~2Q|&C zfCbNNkKF`%Rx$M)eGA@+1U*-=%ae)3$PGH;h6k7$=Qx}Tz3Z72G1_w?V%0HQsXfYR zf*mdrY%64jgA3jXwzdM3bzyUajNtapKs@#P$drL5)bJ-W^MV-xf$*)wUvet3BDCzO zZU4K#*Rn@a_}ky@AkHKelIrOZBT5+}@Ix@NPx!Sp!wG||Ueis4JC-bNq#(gTJ}S7W z{W!%5vIx3CCPhX@MB}l^zCdlHmbw&&R!$TEwzlBCJ*yA~M`s@(%cK}q=_7I< zS;ia-W~K`hQ%h5xi(Ij(B_c&8m>4c9r(ll9A~Em}`P8TnG=Yg=d5SgaiwQTo%O53^ zZA;-^d4T~M0hAl=H5N*r5j_sJ;w(QiXhy*n_NMhu_IU|@8T0F)iXHF35u~1ls z7`*Iw{EA7og>5TGOTgWj+Hj3VaF<;==cDvfcU1h~d`B zFF9eY`ph(RI8^2T+|*fu^tEYrlb$ct`M=XFQj$HRF>nR{@AnS6HU7WX?;dRU z|2k5|*53=RDNiDBs^43Z1u1Hh5KW?)A>ZEs39hA4V`WejJIaaRANEXg8#qO<)yUaf z5wvb^p&+|^L{A8JcdhLL*xa=Ec@S=I3$ZhY5Y8==cXv@^4NXSPz0b{)9*>5`H9KZ? z93?}dgD?Y7DrszSbm$rk_IHuTu+Z-4Z~^6-#j4g}LyZLp-l*2l(wCH5tgu|QBACNi zQTOg{26c7{H|>2UK+B$vt9Z?7hPJST8bHe|`m*Y#tXkWh%$L=7rPUVh_}+@3xrr;T zHksK}mi05MP;FN=h%9SdtyO|BpV}K7jD@&9#~5i7O)i#~mwQC@I3IjCKRuH;q@Suy zYlU}rE!*sFbFlkAlIrrmWJ-B91z^?v-@(paUH~G}%I#OBwlPpu6nG!I+CwPb! zpfZdjYs>VOK)cWFBN2aL;rRv0vWDg`ljOc(sYjDo1;%nH00kl@nIb|t%2`WuhMCO4 zwEb}29lkxeI6FK#nU(lM*DEQ~sFVpXq{xzU99DCEzO{|bke@T|g$`exo}64>ot?iw zJA8e3d2)4p@`n$vuP#n5FWg3&jeK`4G5I#NpYa@0K1Yevdm*eN|cN#GIR&HLjA z5M}{1bOz?nn+R)(Z$jtj@bdE1(BDs2Z{Hs;Y}nE>v87$PHxKaQbIuPl#Axm80QT@u zE7ru|{$keS5~f)}sQl!iwE(fo1X4Y*I;faa~bGIk{dTCx$B1ztJHa!oWKUaLN-jG(i-LAY7ADENg{P6fEF<5?TbWqw&*Mquyf*2{sV@KJ zfW*)32eLx`@9g#Z_4hx$jsM4bQdx%EwI{hA3rT-<4;Qb$SI+&cxEc0JfnfL$tFX+O za}XmLwF1a8rMCA5o4cXe_pnOjGDctH#mg4~zy#%>l(7J(<|g-8F!teXt>w&itL&$j z-o#07^+=%Z#?V2Wy|j1;u%%10*hisO3Fc8RWeg}!C?C1}d5v084VeX9@AK&74-W8y z$VD*pruAa{_w*J3EE47N3yOU35o2ws(KWlMC}hi9RvyLQ-iGJSWil*Xsy=Ftp1O&e z2<)?nDi%T}l;2NIcC61t73(2~FGUA+6GfwS^J#nK0kcxFl`bO;$Mhb=3D>l<%SeB@ zJznbN(W_iBMX%oXzj=MNuK!i#-1BIHEA+qaPOtv{d%wT8dH=VT)cF3dwpSj&svpH0 zAKzbTm;UB=;MYxV|DT^69=|;a)8vcvvC95?uv35k+uPsW*niiOwtRq%KO#%Ph5-r! zmqS%4)`p4+!w_aA3C_Lmoc}hYIny*_I1gfCedGQzCborb`X52?vJmBfd_*}$bD(5y zdMrDI6P}bj{Cu+t24Xlw3AsY4AQjfb&TTaha$Glb6jW;oJ|Xu&iZE^O6}xq!ywl{H4vr*jOYZF$B4j~j>eGSNTLk@P+(^e znNUUNkUTbkpGd6P6mIm4n-gXJ(h?eoa2bdgrQPJfm1`bt%K}L zDYcG}$&{0n#)Ppc=v;E^gserib2CdegW3sAyeoP^2Q&B28hl!vC^$3=AyMT_Z~KKY zbA$Y&X}V;Iwk@GhrIE)|hYHCPs*A;UBc!rM^KVS!B)2}YL13MW?PxrvelwW0B;}c( z;rhl?QBhrq)M91bi%X$*%v7j+;Dg?no)8id=QgO4T^rLgaxDp?zuMWglk&Gj#BhTH zMK%8Y+rsnL8B%t8fGSN2z2I%4l+~CYz-v&51IzM)9&4&A= zCU?m2liyO|cTo`h{PWMj|92R>&~KHv4}a$u0S^KwYz_ee)Py{oZ53ko@Ze?Wx?Y2LQ|bsViYltU=cL zewSUp?W`{PbOSVagwK7#=Pu!9vYs3`r4k?IYg6kcJ$Gv2|IZx_WTpMLU$_4p?Cxyr zziUbVZ1F$7_I&=<(*66t^mO0}>fkESbWd}|9bl4{rmrUqJgZ~ z|NVpA1^usgu+jh4k{%iVqu7X_74^fb6s^a3R+U<>cC0FG`X+}a!!{n{yi|6YH;UEL zFaW;X#Q|uQe0mmmQj@>V{E2)etKIgT)uLW0TJtDYx#BlD<2U-9Lp2@?tcb}1%j2@Z zJ+WEf!SPvE2O)t~fk@zCu|(={76Xzz0#HL>l4d9k!AX{^&+4UqNNf&R8lR(~Ta#gb zuwaQZxT{sOpU;~Xj^fQ;YpZ*!d&awJb-VtP`dgnj`YQf^zx;QP|DSpjxJv)u@6~_* z=b*p8iT|~h^gKQ*4>SBU_?s*@NI244W_Eaf?~`H(&){?Ngj34e*Z=+u%y+0GOh!<4o0AsDQ_V-0B0aGD@`KJ_G#DYa|d309DaUw^k z&wNE&wR)VWX#Zb(zW&+LgZBRuy@6Kk|L(zF-Tv3@?``(~I@06(alV$c+N9@BP5VC_ zBTHRm?q}&^)&Aexsr&!!8I@qO|JRVV;EZUE(*3@PTXgqBjf0#rqboy3;%hQO`@?+J zmqF#375t#a$QX>67&x0B<)dwo$cUyBw7(5i$|pPtwt(Zvc_w_D*@)nnxWo7V`!0m{ z&b`LAG$=ua_UBnJ48r4!s|zjc4`CjOG!-0vI=X;_N)?17syp^yhkg(aew7{jZ;>$` zb<97-r<(B29MFKo*Lh}tj93LPLv@n{FT(-34qk>jHGd09N5RYg3AW%TBB{s~yg5Em yL6}Kl6#O8h36hShF2%= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet +apiVersion: v2 +appVersion: 0.9.15 +dependencies: +- condition: gitops.enabled + name: gitjob + repository: file://./charts/gitjob +description: Fleet Manager - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet +version: 103.1.13+up0.9.15 diff --git a/charts/fleet/103.1.13+up0.9.15/README.md b/charts/fleet/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2f2a4c302a --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/README.md @@ -0,0 +1,30 @@ +# Fleet Helm Chart + +Fleet is GitOps at scale. Fleet is designed to manage multiple clusters. + +## What is Fleet? + +* Cluster engine: Fleet is a container management and deployment engine designed to offer users more control on the local cluster and constant monitoring through GitOps. Fleet focuses not only on the ability to scale, but it also gives users a high degree of control and visibility to monitor exactly what is installed on the cluster. + +* Deployment management: Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, Kustomize, or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy all resources in the cluster. As a result, users can enjoy a high degree of control, consistency, and auditability of their clusters. + +## Introduction + +This chart deploys Fleet on a Kubernetes cluster. It also deploys some of its dependencies as subcharts. + +The documentation is centralized in the [doc website](https://fleet.rancher.io/). + +## Prerequisites + +Get helm if you don't have it. Helm 3 is just a CLI. + + +## Install Fleet + +Install the Fleet Helm charts (there are two because we separate out CRDs for ultimate flexibility.): + +``` +$ helm repo add fleet https://rancher.github.io/fleet-helm-charts/ +$ helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd +$ helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet +``` \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore new file mode 100644 index 0000000000..691fa13d6a --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml new file mode 100644 index 0000000000..5332562fbf --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: 0.9.21 +description: Controller that run jobs based on git events +name: gitjob +version: 0.9.21 diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl new file mode 100644 index 0000000000..f652b5643d --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml new file mode 100644 index 0000000000..bcad90164f --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrole.yaml @@ -0,0 +1,38 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitjob +rules: + - apiGroups: + - "batch" + resources: + - 'jobs' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'pods' + verbs: + - 'list' + - 'get' + - 'watch' + - apiGroups: + - "" + resources: + - 'secrets' + verbs: + - '*' + - apiGroups: + - "" + resources: + - 'configmaps' + verbs: + - '*' + - apiGroups: + - "gitjob.cattle.io" + resources: + - "gitjobs" + - "gitjobs/status" + verbs: + - "*" \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..0bf07c4ef8 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitjob-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml new file mode 100644 index 0000000000..7771db512c --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitjob +spec: + selector: + matchLabels: + app: "gitjob" + template: + metadata: + labels: + app: "gitjob" + spec: + serviceAccountName: gitjob + containers: + - image: "{{ template "system_default_registry" . }}{{ .Values.gitjob.repository }}:{{ .Values.gitjob.tag }}" + name: gitjob + args: + - gitjob + - --gitjob-image + - "{{ template "system_default_registry" . }}{{ .Values.gitjob.repository }}:{{ .Values.gitjob.tag }}" + {{- if .Values.debug }} + - --debug + {{- end }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml new file mode 100644 index 0000000000..51f9339509 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/leases.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gitjob +rules: + - apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gitjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gitjob +subjects: + - kind: ServiceAccount + name: gitjob diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml new file mode 100644 index 0000000000..bf57c1b55c --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitjob +spec: + ports: + - name: http-80 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: "gitjob" \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5f8aecb045 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitjob diff --git a/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml new file mode 100644 index 0000000000..f7c79214de --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/charts/gitjob/values.yaml @@ -0,0 +1,27 @@ +gitjob: + repository: rancher/gitjob + tag: v0.9.21 + +global: + cattle: + systemDefaultRegistry: "" + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + +# PriorityClassName assigned to deployment. +priorityClassName: "" + +debug: false diff --git a/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl b/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml b/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml new file mode 100644 index 0000000000..3fd0b15cf8 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fleet-controller +data: + config: | + { + "systemDefaultRegistry": "{{ template "system_default_registry" . }}", + "agentImage": "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}", + "agentImagePullPolicy": "{{ .Values.agentImage.imagePullPolicy }}", + "apiServerURL": "{{.Values.apiServerURL}}", + "apiServerCA": "{{b64enc .Values.apiServerCA}}", + "agentCheckinInterval": "{{.Values.agentCheckinInterval}}", + "agentTLSMode": "{{.Values.agentTLSMode}}", + "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}}, + "bootstrap": { + "paths": "{{.Values.bootstrap.paths}}", + "repo": "{{.Values.bootstrap.repo}}", + "secret": "{{.Values.bootstrap.secret}}", + "branch": "{{.Values.bootstrap.branch}}", + "namespace": "{{.Values.bootstrap.namespace}}", + "agentNamespace": "{{.Values.bootstrap.agentNamespace}}", + }, + "webhookReceiverURL": "{{.Values.webhookReceiverURL}}", + "githubURLPrefix": "{{.Values.githubURLPrefix}}" + } diff --git a/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml b/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml new file mode 100644 index 0000000000..164340c444 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-controller +spec: + selector: + matchLabels: + app: fleet-controller + template: + metadata: + labels: + app: fleet-controller + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLEET_PROPAGATE_DEBUG_SETTINGS_TO_AGENTS + value: {{ quote .Values.propagateDebugSettingsToAgents }} + {{- if .Values.clusterEnqueueDelay }} + - name: FLEET_CLUSTER_ENQUEUE_DELAY + value: {{ .Values.clusterEnqueueDelay }} + {{- end }} + {{- if .Values.proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy }} + - name: HTTPS_PROXY + value: {{ .Values.proxy }} + - name: NO_PROXY + value: {{ .Values.noProxy }} + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_DIR + value: /tmp/pprof/ + {{- end }} + {{- if .Values.cpuPprof }} + - name: FLEET_CPU_PPROF_PERIOD + value: {{ quote .Values.cpuPprof.period }} + {{- end }} + {{- if .Values.debug }} + - name: CATTLE_DEV_MODE + value: "true" + {{- end }} + image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}' + name: fleet-controller + imagePullPolicy: "{{ .Values.image.imagePullPolicy }}" + command: + - fleetcontroller + {{- if not .Values.gitops.enabled }} + - --disable-gitops + {{- end }} + {{- if not .Values.bootstrap.enabled }} + - --disable-bootstrap + {{- end }} + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + volumeMounts: + - mountPath: /tmp + name: tmp + {{- if .Values.cpuPprof }} + - mountPath: /tmp/pprof + name: pprof + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.cpuPprof }} + - name: pprof {{ toYaml .Values.cpuPprof.volumeConfiguration | nindent 10 }} + {{- end }} + + serviceAccountName: fleet-controller + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: "{{.Values.priorityClassName}}" + {{- end }} + +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml b/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml new file mode 100644 index 0000000000..17d1ba7864 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/job_cleanup_clusterregistrations.yaml @@ -0,0 +1,40 @@ +{{- if .Values.migrations.clusterRegistrationCleanup }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: fleet-cleanup-clusterregistrations + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + labels: + app: fleet-job + spec: + serviceAccountName: fleet-controller + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + containers: + - name: cleanup + image: "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + privileged: false + command: + - fleet + args: + - cleanup + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + backoffLimit: 1 +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml b/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml new file mode 100644 index 0000000000..361d68c08b --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/rbac.yaml @@ -0,0 +1,114 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller +rules: +- apiGroups: + - gitjob.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - fleet.cattle.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - '*' +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + namespace: {{.Release.Namespace}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fleet-controller +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fleet-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fleet-controller +subjects: +- kind: ServiceAccount + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-controller-bootstrap +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-controller-bootstrap +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-controller-bootstrap +subjects: +- kind: ServiceAccount + name: fleet-controller-bootstrap + namespace: {{.Release.Namespace}} +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml b/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ba27c748d7 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller + +{{- if .Values.bootstrap.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-controller-bootstrap +{{- end }} diff --git a/charts/fleet/103.1.13+up0.9.15/values.yaml b/charts/fleet/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..7da936a818 --- /dev/null +++ b/charts/fleet/103.1.13+up0.9.15/values.yaml @@ -0,0 +1,87 @@ +image: + repository: rancher/fleet + tag: v0.9.15 + imagePullPolicy: IfNotPresent + +agentImage: + repository: rancher/fleet-agent + tag: v0.9.15 + imagePullPolicy: IfNotPresent + +# For cluster registration the public URL of the Kubernetes API server must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# For cluster registration the pem encoded value of the CA of the Kubernetes API server must be set here +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# A duration string for how often agents should report a heartbeat +agentCheckinInterval: "15m" + +# Whether you want to allow cluster upon registration to specify their labels. +ignoreClusterRegistrationLabels: false + +# Counts from gitrepo are out of sync with bundleDeployment state. +# Just retry in a number of seconds as there is no great way to trigger an event that doesn't cause a loop. +# If not set default is 15 seconds. +# clusterEnqueueDelay: 120s + +# http[s] proxy server +# proxy: http://@:: + +# comma separated list of domains or ip addresses that will not use the proxy +noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local + +bootstrap: + enabled: true + # The namespace that will be autocreated and the local cluster will be registered in + namespace: fleet-local + # The namespace where the fleet agent for the local cluster will be ran, if empty + # this will default to cattle-fleet-system + agentNamespace: "" + # A repo to add at install time that will deploy to the local cluster. This allows + # one to fully bootstrap fleet, its configuration and all its downstream clusters + # in one shot. + repo: "" + secret: "" + branch: master + paths: "" + + +global: + cattle: + systemDefaultRegistry: "" + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +## List of node taints to tolerate (requires Kubernetes >= 1.6) +tolerations: [] + +## PriorityClassName assigned to deployment. +priorityClassName: "" + +gitops: + enabled: true + +debug: false +debugLevel: 0 +propagateDebugSettingsToAgents: true + +## Optional CPU pprof configuration. Profiles are collected continuously and saved every period +## Any valid volume configuration can be provided, the example below uses hostPath +#cpuPprof: +# period: "60s" +# volumeConfiguration: +# hostPath: +# path: /tmp/pprof +# type: DirectoryOrCreate + +migrations: + clusterRegistrationCleanup: true diff --git a/index.yaml b/index.yaml index f82de51200..78c6f6ee5f 100755 --- a/index.yaml +++ b/index.yaml @@ -1292,6 +1292,32 @@ entries: urls: - assets/fleet/fleet-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/auto-install: fleet-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/experimental: "true" + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: clusters.fleet.cattle.io/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T21:49:59.629911312-03:00" + dependencies: + - condition: gitops.enabled + name: gitjob + repository: file://./charts/gitjob + description: Fleet Manager - GitOps at Scale + digest: ffc5a0eb1db89082a5544cdc8da674861a8afb1a7c769310c7b275062bd6af46 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet + urls: + - assets/fleet/fleet-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/auto-install: fleet-crd=match catalog.cattle.io/certified: rancher diff --git a/release.yaml b/release.yaml index dc64453c0f..f379deb4af 100644 --- a/release.yaml +++ b/release.yaml @@ -6,6 +6,7 @@ rancher-logging-crd: - 104.2.0+up4.8.0 fleet: - 104.1.5+up0.10.9 + - 103.1.13+up0.9.15 fleet-crd: - 104.1.5+up0.10.9 fleet-agent: From 4755590445095ec1b2e46566476cd4a90aa4a8b6 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:51:21 -0300 Subject: [PATCH 14/19] fp: fleet-agent 103.1.13+up0.9.15 --- .../fleet-agent-103.1.13+up0.9.15.tgz | Bin 0 -> 3204 bytes .../fleet-agent/103.1.13+up0.9.15/Chart.yaml | 15 ++++ .../fleet-agent/103.1.13+up0.9.15/README.md | 8 +++ .../103.1.13+up0.9.15/templates/_helpers.tpl | 22 ++++++ .../templates/configmap.yaml | 13 ++++ .../templates/deployment.yaml | 51 +++++++++++++ .../templates/network_policy_allow_all.yaml | 15 ++++ .../patch_default_serviceaccount.yaml | 28 ++++++++ .../103.1.13+up0.9.15/templates/rbac.yaml | 28 ++++++++ .../103.1.13+up0.9.15/templates/secret.yaml | 10 +++ .../templates/serviceaccount.yaml | 4 ++ .../103.1.13+up0.9.15/templates/validate.yaml | 11 +++ .../fleet-agent/103.1.13+up0.9.15/values.yaml | 67 ++++++++++++++++++ index.yaml | 19 +++++ release.yaml | 1 + 15 files changed, 292 insertions(+) create mode 100644 assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml create mode 100644 charts/fleet-agent/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz b/assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..5beb26afef8ce6f5496c869127da1ff8c1507b1f GIT binary patch literal 3204 zcmV-~414n*iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI|CbK5wQ&S(CLF1ofd$(huft<19SuFB^)d)afmDcjkrt86L@ zku3>p5a0lyJX($Z`xRa&k(6vD8IL_PtNvh1Km%xijRy~!fW{0_M`VIrcTT57>Ud6a z_Uu8W+wFFTM@RO5x7%(1?{$Yq&w59%j*o|f-qG;*S+_Sh>K;FX?!%g~T}q{ieAfNh zZM8f18xg{}&_q+gRT2QCL=z?^aZ0pi7*o+nQEED-m?a z80F=UTXPGPIn^p!X&v5Do{3wvRoi#GX?#Ea-rL@f?=f8>L!vM;T}Yrd-2zh3zg88_ z`T;45)z@zP$GCSKWT;X}3vJWhn4<8Ga56y&C#DLB;7?TlrBFaLT&09zKvSbT)w)ng zr<0l)tzvJ#F^4FHT6LJ12-UIH#A-GPHmzIrwbvcQy|_2{x3chNJ+;s7<>91`|1-i$ zR1d!Z?7;uQaM*3*|M2M59{(RhXig?*kl!F-=t1z%LZU%&iG@&93proGf2WQT0Gdn^ zn0aIf4&Zu2qj023Qpd?1%+~k5@g9a?b496NUFZ3L!2C9H4Rpyw1o>JAW^ZNYCs3A!}$gP?vP!er=;1uA<0R@OQ{XhKDdFO!-_b zX{ud%99S!_&#&Hz3=;_JMOV+qw*V3+RB371h1-b=~z=#HC&Q7OhdvVDsJ8U^6e)Y-d|dsol~ejcs@ZvksYI;; zH{h4GtcyFiR8>z!s)7S}Z>bAp!VX?KC4Ay$(+HWkHIvi&OvxE41#?k?M8eqI`ZL^4 ziMENRqnV(55>T#Da>5MYxMuFMX2tBBCuWz9MqO<1lxlcHhF8ci)j|T{#yuFi4l5Xo z`jpAAU}A23JFpd5DrGD>IDiY0Avkxir3#Um$4n~iBxq; zg)*9yG{cUuDpJ-gDtO1151?Ue0*j@q_nazi+{EO9dBWQIy6*_WcM^Xnsl*1VSQb5e zVSsqu;RHT@3XEl`X1}kF1ppD;;5@OF#VHeIR!A|U8Ope6a>*&Dni59;iCF*u9tw?R z;n@!*VWCa6KvUbe0N{9xsZQWRT%}XYN`^r&VPfR`*12h7HJz8f#xW_G_LFRGXDvW& z{I{>8Ucu(ORkRoPd+~7)WH>4(iNjR)+lu;Wt%)hFer$!nHlfvyIQeV$pdS{I4Uj>*ba*W?6&poP!~YgqG`CI8ldz zQE`i#ORldXTEMN4H%yRBL27B2VW=J=JRuPZ>oo>1JKnx(kZopU@v~OB>ywAJupY%Q zP*OqEik4Rh6|N!5ByR#-Yh<>Xp*&za8=4@xj-u3RdTO8D%frcb{7(fR(@9Q>2MGr3 z!2j-1Z`j8F!9M=`IC4XImcXff-tS1!3`+Kb95u;^CWggq#t5q%=Pn8S6j?H}uzvv- zi~5S;$>(zEb3>O9CX0m@e^!FGG?&Z6bv*`o)+MW*R+xl~MOCUwTrR_^U|8RMh6&WP z>p9D%<9*A2!N}(J`M=&GJp=&knE(CbSMB)U@VLL9|BoW|`A&*Lb!NT5VZ!V6Hu-)O zPz9!TLtkzD>}1M`PN(NSu;Qj8DT-!EfXd~3nOZwWSflvX3IM9+jrhyVbz*iEs}4yo@?ASg`&tN zZl*-cY*B0Jiv|2p3XRR$>_)a;<%-|e(iEnpqpeJDV`8Bnosw>;9A|(YGQARaI)qMP!6h$*7f|K*} zX0PgNmO?qBDW0UMD7oGYWBKRTjGruk@|2a?U3@);pMX=IA=mIdY;Mxl;jOq4AG4aE z#s863Xu{C+b3Q<4!Fn zvRNM#$^?4dZnv5KCn?IpXVfftYTU16`})sOo2~BWLNJ=nKif%X{(c)9*v0<`!+vx9 z@Ar<5_VJ%bktm89oZmlr$|v#7_cjEuA_NyM>%H}fEu6qFXOZ<5^VjG2@cv@4nse1_ zfVlMzz)Xy4IWI)kiX>A$k*ItO*rs4R!TIa;yg;A&)|IiJ&)mnS9`Rq7Y@h#yA#W>w z;@2+X)~>I|fL;8*d)%J?{r-ObKaRBL|HwS~^_>5+7;Ty9);+2hbtC|2ogxDWr^xbH zO*>N|Zo&i#p>#yK(uA=?_pK}@l4Q849%aZ-W8@}tn1D@GWtt*pm>t3h$3kLclT(Sd z-4$*$cNF!_!Ix1miN+zcLpcL_Gkf9BF4^-1+|Zc+iUf9ryR={~kx`YrlQ!q|{R(=|3$% zZ0rF}-TTx_!SMd5pg+J9)sv(tB}-<=c?6`Oc5m#N`tz61?Ri5bN||CKafWhKONf9A ze(4oIT%P-^@MYN4H3@Bc^L1VPFUm8@C*Ri7Xyi+qFFt-)_4yt{0)Wk-{+a<+<>()n zYCFcRrsfj7HHO>Tsm1=dRlHhme?=jSy*!|7$A5Qv@S#?KUGbmeLBEav!~SrO|BoT9 z_q`R$-Fs{_Tr5VzBjo9dTHd4X_9a_Rwr*6?)^R1P6TlnlHpA((4t{T_*7I8>>@n-R z3!I)9{R&3Sq}Z_-jD*l;42s_fT;4VRe_q6=DZA!>uiKvguLgVm_c+p;|62*e{jYzY zx(9ptIkJ8J&j_O#(fBY7VCVjCuyOvkH`vF2A4L`mItGrg$@stMZOvi_ zo>B$&+Ue>Jmk;}529(=t80)Jne^5Y_P4m}eZC)K-!kuQkaCRWi+!@y`{n@Nz^H#}I z^=8wOZ5^t;u&$Gty3t`HgDUX1D(t~zca`qY-i*nG(!l5qvN;f@M4b$U^|qhdH}~?( zOCA3&&rV*yJB#z|f!o-H|Gj=Y{?~ss80_)?G34OD-)eO~DDdCNa@e8Rp54)o zP}#+xY5pOMN)7f^D5|YD9TgN(OeD#3qG?JPo5w+Ly?Pp%NK2#U?v9#jk!$I$&Q+uP qF%v1=;z&`A_jK(={M0VJm%Z#|FMIg{`Tqa_0RR6#)M<7AOaK6H6JsF& literal 0 HcmV?d00001 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml b/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml new file mode 100644 index 0000000000..5ebcf31364 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet-agent +apiVersion: v2 +appVersion: 0.9.15 +description: Fleet Manager Agent - GitOps at Scale +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-agent +version: 103.1.13+up0.9.15 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/README.md b/charts/fleet-agent/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2c5724dcef --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/README.md @@ -0,0 +1,8 @@ +## Fleet Agent Helm Chart + +Every Fleet-managed downstream cluster will run an agent that communicates back to the Fleet controller. This agent is just another set of Kubernetes controllers running in the downstream cluster. + +Standalone Fleet users use this chart for agent-initiated registration. For more details see [agent-initiated registration](https://fleet.rancher.io/cluster-registration#agent-initiated). +Fleet in Rancher does not use this chart, but creates the agent deployments programmatically. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl b/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl new file mode 100644 index 0000000000..6cd96c3ace --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +kubernetes.io/os: linux +{{- end -}} \ No newline at end of file diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml new file mode 100644 index 0000000000..f3e83a89cc --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/configmap.yaml @@ -0,0 +1,13 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fleet-agent +data: + config: |- + { + {{ if .Values.labels }} + "labels":{{toJson .Values.labels}}, + {{ end }} + "clientID":"{{.Values.clientID}}", + "agentTLSMode": "{{.Values.agentTLSMode}}" + } diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml new file mode 100644 index 0000000000..582eed608d --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-agent +spec: + selector: + matchLabels: + app: fleet-agent + template: + metadata: + labels: + app: fleet-agent + spec: + containers: + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: '{{ template "system_default_registry" . }}{{.Values.image.repository}}:{{.Values.image.tag}}' + name: fleet-agent + command: + - fleetagent + {{- if .Values.debug }} + - --debug + - --debug-level + - {{ quote .Values.debugLevel }} + {{- else }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + {{- end }} + serviceAccountName: fleet-agent + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.fleetAgent.nodeSelector }} +{{ toYaml .Values.fleetAgent.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.fleetAgent.tolerations }} +{{ toYaml .Values.fleetAgent.tolerations | indent 8 }} +{{- end }} +{{- if not .Values.debug }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml new file mode 100644 index 0000000000..a72109a062 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/network_policy_allow_all.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ .Values.internal.systemNamespace }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml new file mode 100644 index 0000000000..aad4eea415 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/patch_default_serviceaccount.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: patch-fleet-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + spec: + serviceAccountName: fleet-agent + restartPolicy: Never + containers: + - name: sa + image: "{{ template "system_default_registry" . }}{{ .Values.global.kubectl.repository }}:{{ .Values.global.kubectl.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", {{ .Values.internal.systemNamespace }}] + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.kubectl.nodeSelector }} +{{ toYaml .Values.kubectl.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.kubectl.tolerations }} +{{ toYaml .Values.kubectl.tolerations | indent 8 }} +{{- end }} + backoffLimit: 1 diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml new file mode 100644 index 0000000000..1a7e8d8841 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/rbac.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-agent-system-fleet-agent-role +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - "*" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-agent-system-fleet-agent-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fleet-agent-system-fleet-agent-role +subjects: +- kind: ServiceAccount + name: fleet-agent + namespace: {{.Release.Namespace}} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml new file mode 100644 index 0000000000..4715882047 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}" + clusterNamespace: "{{b64enc .Values.clusterNamespace}}" + token: "{{b64enc .Values.token}}" + apiServerURL: "{{b64enc .Values.apiServerURL}}" + apiServerCA: "{{b64enc .Values.apiServerCA}}" +kind: Secret +metadata: + name: fleet-agent-bootstrap diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml new file mode 100644 index 0000000000..73e27f0be9 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fleet-agent diff --git a/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml b/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml new file mode 100644 index 0000000000..d53ff1c508 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/templates/validate.yaml @@ -0,0 +1,11 @@ +{{if ne .Release.Namespace .Values.internal.systemNamespace }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.systemNamespace) }} +{{end}} + +{{if ne .Release.Name .Values.internal.managedReleaseName }} +{{ fail (printf "This chart must be installed in the namespace %s as the release name fleet-agent" .Values.internal.managedReleaseName) }} +{{end}} + +{{if not .Values.apiServerURL }} +{{ fail "apiServerURL is required to be set, and most likely also apiServerCA" }} +{{end}} diff --git a/charts/fleet-agent/103.1.13+up0.9.15/values.yaml b/charts/fleet-agent/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..ae03ceba32 --- /dev/null +++ b/charts/fleet-agent/103.1.13+up0.9.15/values.yaml @@ -0,0 +1,67 @@ +image: + os: "windows,linux" + repository: rancher/fleet-agent + tag: v0.9.15 + +# The public URL of the Kubernetes API server running the Fleet Manager must be set here +# Example: https://example.com:6443 +apiServerURL: "" + +# The the pem encoded value of the CA of the Kubernetes API server running the Fleet Manager. +# If left empty it is assumed this Kubernetes API TLS is signed by a well known CA. +apiServerCA: "" + +# Determines whether the agent should trust CA bundles from the operating system's trust store when connecting to a +# management cluster. True in `system-store` mode, false in `strict` mode. +agentTLSMode: "system-store" + +# The cluster registration value +token: "" + +# Labels to add to the cluster upon registration only. They are not added after the fact. +#labels: +# foo: bar + +# The client ID of the cluster to associate with +clientID: "" + +# The namespace of the cluster we are register with +clusterNamespace: "" + +# The namespace containing the clusters registration secrets +systemRegistrationNamespace: cattle-fleet-clusters-system + +# Please do not change the below setting unless you really know what you are doing +internal: + systemNamespace: cattle-fleet-system + managedReleaseName: fleet-agent + +# The nodeSelector and tolerations for the agent deployment +fleetAgent: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: [] +kubectl: + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## List of node taints to tolerate (requires Kubernetes >= 1.6) + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: "Equal" + value: "true" + effect: NoSchedule + +global: + cattle: + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.21.5 + +debug: false +debugLevel: 0 diff --git a/index.yaml b/index.yaml index 78c6f6ee5f..3382832577 100755 --- a/index.yaml +++ b/index.yaml @@ -2230,6 +2230,25 @@ entries: urls: - assets/fleet-agent/fleet-agent-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.23.0-0 < 1.29.0-0' + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.9.0-0' + catalog.cattle.io/release-name: fleet-agent + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T21:50:58.025196698-03:00" + description: Fleet Manager Agent - GitOps at Scale + digest: b79e59689ae444b4ae5693db5266bab2437937bfdf60aace8845a653e222a0ec + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-agent + urls: + - assets/fleet-agent/fleet-agent-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index f379deb4af..e1e3dcb594 100644 --- a/release.yaml +++ b/release.yaml @@ -11,6 +11,7 @@ fleet-crd: - 104.1.5+up0.10.9 fleet-agent: - 104.1.5+up0.10.9 + - 103.1.13+up0.9.15 rancher-monitoring: - 104.1.4+up57.0.3 rancher-monitoring-crd: From 2781161a5b0da86f14358d9f26bd00e94231371a Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:51:58 -0300 Subject: [PATCH 15/19] fp: fleet-crd 103.1.13+up0.9.15 --- .../fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz | Bin 0 -> 50477 bytes charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml | 13 + charts/fleet-crd/103.1.13+up0.9.15/README.md | 5 + .../103.1.13+up0.9.15/templates/crds.yaml | 6859 +++++++++++++++ .../templates/gitjobs-crds.yaml | 7690 +++++++++++++++++ .../fleet-crd/103.1.13+up0.9.15/values.yaml | 1 + index.yaml | 17 + release.yaml | 1 + 8 files changed, 14586 insertions(+) create mode 100644 assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/README.md create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml create mode 100644 charts/fleet-crd/103.1.13+up0.9.15/values.yaml diff --git a/assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz b/assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz new file mode 100644 index 0000000000000000000000000000000000000000..d0b7ed5eb4f11130986462e526d31d70f0ec16f9 GIT binary patch literal 50477 zcmbTdQauOywr$(!vTfV8jniwLfA43XeR<|ZX3ogSc^f&# ze1jko3JvH#&kq$4jlQH3qoJf6yR17Gn*oa|qmeS3`5$F2_TQ?i>~d;W7Wy_u?f^ww zK1pLMYoN<6YuC+|W(FUf59Qs8D7G@Yl0OT+ukl{F)&34{b` zFG${@xmwSMuRiGRLL3lhM*LNhTcWy1KrcjHJ%O)hI&y(Yxa9Ad74@<=h?6zn%fEa+ zpRaeex%|OD{63#;A3GyGU+0lOZugIs3_YJ$7{7DApDTadJ`W@RZCv^0jeOus>e3YV z?+f7Xd}wZsc@x9445OCeo@qmXf(dk5`hje<}dVln$>rB6|CD8+Zapws8(M>X7g-_(L5bE!C}@;N(5XpG+& z-7Ewcfypp047i0h3QJVFwa9$jL!e@&5Sp)Cmz%=DxSyxl8G8xfo3@7nI1W6a4+ z-bO$<=|~SfaERV3ksb!nQWXkLO0drTu1%&sY+P6nC7|bOMOz%`Q$iv8&CM!II{6t(WFQdm z`OKVt^L<}hs4aQ;L9%AgY_<{9Qn=SaF8Lul;n0=wR@>rtcM{Qfx4LcjOhOG-uBuBR znnZ7*)X0cWLc`iQ45H%VNY!#C5=a1-dVHYio9N$mkIt2oAWSAS;{Sm>eJD6b^`UN1 z+O0YXysa#+7eGmT}ltLcsvVbc&h}&Qx0rh!Zu=d@Ed@5vYL{S-qC~f^RL8v=35DNLE7j zXB0`23f2%t(rwX^v`cAT1|iz~7I&p~bZU9xAceX#Eg@C140v?tD_z8<`EaUKo?a*D zlgyQDFOp(A<*8T>qdxm)tjqImoMQg#HCRHDgM4Q6&!B$Try;bo|W&pQ*Z*M7j zY^0wkRJao`CAIMS7rE487bt7gLMfClumxE6BI$l2N&!rj&s?fC660(%$+Nal^2ZWy z(F=dd_$i*m7Y_V3G*vcm{6&2mmKaRVo<|%yg#u3F^Agzj^_v5(FmxP*kc?$?Xt-!9 zHlyQjc^R-&G8cka$W99YTB8d|n+cbI`6akWBRO;_0M#fJ{V80@$fV8qf-WF)x9#Cb zhd+1|eNVglhDzpXyZ70d<*!HbV=gP+4sVZ-=2d@n_`rCEm?bXP%)9x#QVVH+XFD2i z=nPWIuPkO`SJovSwG`Oud19e&9hCCJ=N0#v)M=qeNO6^-_q<15$FW(iX(^W&GWXY` z#Y%EhkMMnrCMspC9zE5(r$kiB4C~X;ixj(juJ_0 zZB|D{ik4D|pzF4_^q0^6aoT~%C|(iiq>apJ0pu;Hg<9KezwN)g95d}i8_=_4ULCeh zWyBu=4^g|;IwczkkwK#HyP995e-ddW0!-IHC^(L5(x>uJ2ls@WBpTu9-beK_{!U-D zMU%tUQr6GODk3@B6J{AlTLv?GA42dnhZJm`*@;Nr(Uq(blmoP^3MlWrKk!aPh{;%u zUs33omSB&=nx`{(RDQ`8$mI46JmD1VG1>m5L{Ildw=uvc4bd0 zFQanOWqWZKFP>iv)c^qQo+EZ+iUTl@(7;xYbp@|^JK9foRGlBJ>ins2bDf9RK5LSH z@0<8V1y$V`$)-2;uS55rhFHl9md=5g2!W{*av3&zKiDRMA4;x{aTtMisZ-T?I=gCa zo@csQ;VoWb+OVgD(>Pulr_{<7T zg-xn&8jo{k`osHfe<=WZ0PV#G1!;b=4z<@_zx?|BOS>AjV+_MSxV26h1mABL{+gRdcbSMQ z(t5D1cjhqxJG!5=&2H}Iz9WQO{V)LZr~`2K;G&kW%)88w-i9TFakPDNq>IENG_%a( zg&&+lBu&8Jx;Ie@TC7E{Tw0Ikp+IG%nd|a#x8kPepidS3Q!_iSP80C3{u)Jc0EFR8 zHOyM^`!XeT@~3>;C2S9R=b-q}+z2`Gcl0Z~u~RXa&efmlP)q0;vd$Bdgpo=bqS52# z#ZP$B={iHl&EC37+t`lPj=9ve`NlnDYa$avs#oRmUGXlP-j7idbVQB|s`YQijDmPg zNYFY`4MiXPpv^6Pc+%q)lAytZC#5t4u?&^&b6iiZu`E^ZEyF{iA!TLmn!;I`tQF05 zO~_fN7twg?3324cP*T$?v>FsoZeI%?QVu*-L^E`jlu&LNf-%t4<;MFIIbY+rZg7Gw zYA3a;qPs+})J!U=VxSs3oYYxo+#CXxd`3b9*7ak=^Bw6sJa9!^4bdo|Bc!Ot!iZ07 z>9w+Y2`A39WVcqVexqdf{Ec&bB>5H9u=E3hy31I)2#Ic*u+h2~XXAEe@T9Qxbq%qd zM4xK!STIrCsMMZZLNk2Dm-2unOhco4wOchAZ#S&uJ++rrmsA8rLn>Wj#y_g^>>@zQ z%TPyH+x&Q%+QqY?_rONtMXZ(0f&>XZ_ExQn5NNnymrG`uu-O~VW+Yohv%Vk|#LmI# zz+yvJ^Bm1v)P1dgYdC2+fN<3tg@bQ1L2^-(6~%dqwPh25(-0g@MB==GoWUYmXJ;27 zVRlbv@p<;Xlz;wJ%{Kzi9+-jsbMkS)O0 zGy2Vw<|2XPifk2{ef<3AhQxFHJQ_TrBF8w3Vn5#mx}+n!ZJlxXGpG>^7gsLdZt!9sQDR z?7mFsE8ggDBe&{Z*%Oewg1Bv1@?AZq4v05i4xtdt_G2BvpWuOr6DJx;8@mS6Th4^R zVYC^O640i}TbQkCXqHE)wi_7Y6s^PEdc!hWRgls>K8yx`VDG|G2e`NDyeJXegn!qN z*@zicQ7You4h{w+jMJsJVpryXsxp8<;4gJs(JuQ>dL+!!>#bLsNdapkoK;8Wm<7KP zgF}0GyQ(DVt+k;gXVy3R3bpoUI1FfcJsR8(*8nE03!O=1?)qQws6n>RSP}wfQr|(p za7=GHdioaS-7kBAiaxh9j|( zg&n(>KWbT2mj3XAgPMTcoq9YNv=9fRZl~^(Cmp;PTCXuwsws#QS**KB?<{mqBGWv8 zBmYT2KTHY8T;O2|x0R~kN@khAxGHsZa` z*vjSq`7-@+z5e=o`1w5Yc5g{RD{zf1QOz~pi?!Y9?rxE1{*p9%S)C}a_ibuF=RyAH zDm#Rylg+>gN^=(*p(ECM>AXzKyKbB1nJSufE4SNl^9-g+tPqOjyK@Y-9j)XM57JVu z4cN%o+c?&w9N`en7+PAe+(`dAk(V00!L;xb+VmA?rRW5#knYh&_gL?==s0?+yOuE2 z@W`S=ZDlZdmI5>_uV;D&6a%c6zXHQ8sp^`DQM5=-Q35<+>B*KjYWAlkFby*Pl|dsB znOhb`+PR38f^aVV@71mNQ1&vd?%b^;SYk{?(P^!Ot14EFmd0Ya#wVYN$w$qMXu?m# zC!VXiTe43nd$37bHzUQk39;1`aS2TDYB*288qGTlf7w$3@I6cpD4sD(2y9gvw5d${ zxcwdlebIV?C{nYU=@wH_;f5O+4=pNwZ2B(+Nz=IeJPsDoYUG3$@_dJwX>h4>i{jDh zi6P#-w=-s%9g4H^@At{z&TM+-ZcgtkTIlchN(KzwlHa-XP~XRy40i!uPeZmjK3|8K zrMjQTcWSWB))40rm?H|Qs>?g#I?wqoD7X~AuA#g51J z`-6ka0`Ug9T+d+NxY8vodX5ggmh~2Lp{+UOi4MbeKN1;cCiY1tjGibM&fs&SU5n_I zPEP6^qqDag%tL1kZK|pNdc}3O($Y(2HUKH^F(S%Odz=*RM17Of2+~qgc$Fbv7^OjV z^;5Sdv_eIRR&jU@M&8zPpC8$pL8zXbnQGkA89KJz?frXH6sOXql^ox(S_sL(>B*FL zWat++ARpO2_&T|zMa{LCqM6V|r<)}xilLIFQUL6}tI6LbRKx2opo$ z)B7R%T4!a8O{+C$OYuOw7vDcE;<@`pehe0EPs`L^|8hqyrv>xpVK+#LTB{{FdxSBw z!@Qptn4Az(`5HqAe*jVGTrTw5>>EJnHy^M1!Wxr6HlsxBCy8DR>b1kwXI0dF1~jiq zHw4nsItM1=+1(d-LP^^5Q_13Bk^P6H>OrX z!`q6Xxnxju8rmL5FP3T8kp`z$wiei(MsptyW)$(bq-NI5UGPWPf4rXW#j-P{Q_GUx zYtUa2`hHu;k@^pXfaB<>bU`|35e|iF;Pj9Y58$}J|K3{b$?7Ql>y67r=LfRi??(vt ztdYr#;JeasH=_Gmdx>M+MwH*7t(zI}v*0d8XRkxc3^E1I=dB-Cj(h%kkE{CCtSvU{)XZ*FKM!avcr%JAvlRZFsfR(n8vL6<+ao}`&N@IlO%KB*bLIcaXu;3b z!{Gk6dusmg`0WGgrEsfU9jrgr?>g8Vs{i++J>Zx9K5K_TPK-o&vXYRf^L;^1vsV8m zJ>UPX|AiN=rF(MbYXx#1FQCuzEEW_mn$F7P{#UMti7J2kds0y^qJm&-^hys zbJ+iLQkYwi0Pl&SDDdA&Q%Tf$e{9n&&LnEtr$Aw8`+?>WLmfk}BEpi*2Y}57muCf5Wd2E#wC4 znDl+`gL9sE1XD1X4q7ci7}_;NjIuMI@lw!Lhd^ZsDU|kAE=PaVkJF~W`ltpG)>Dy< zsYsa{(51us0TZgQI`5NMCf`da0==@}ee;Bf0oNYr!G(hsAyg>#4Jc zjY;p6PS={Ud=_I0#0hA)$F490ANqg5FW*S*?C6nX&3}AgUbYSRp2p#3+aJ3L?udTM zOmKRN*tP!^AZfRFY7*cdY^&&M4Qn8!QT7{)GtN06lFf^a*5qB3`q98w>X;wMjfj*d z7PhN}rDjLs7tR%`*SIGLH{?uuy+BWO`82V^Z$3Zh>QwN85Lt!RQJUehmeIL^gCh>H zf$#bAK$~1<^ms$2N0MRIi8|5p+35XpeJ?zX{0=H*M+M#ffPJep9TSW0;rz5e14IX5 zn~TY&ix@$ar3e=SBVu~GCIG^d2!>IC_FV)vj;2peZYI|tEL;JTP+S)@Tj;MF4Zf0> zo{xSg0RMz8}MIZVJXJ9$DikVk*3!N_F*+yL~5` zAplO#M3o4$t7o5dk2qKoT*vV9FRNha(@HC}L2x|%X+>u0!<+4tm84{u35uY^+v*|w z1vT#jY2EMsDG9<&+yuliLQ-*~m#;M^P?4KA7ShP#>ey3~8t}5^opCY5%7$N3k2Ah} z8B->)*dvkzRR0`9h_#i6%-Uk9YIm@YxqYVgM8tB@VW`WPu5Sn{Dx60ZI?ouQM8I_F zuJP|WC9HJiZ^>U^y>zMpsY5J_Uu)O6EG{fi17Q_$eyJsMWHZAqIS zSjDZVn;R?w^6hE|1TAYOudneZR-1L8iRd;Px~*-(XWuRR)uO%&-?N-Ja#LK30#L~@ zr=xn?tZ7H8PN=8uAfO8*0;HE;gJgi$Jod;iz!X~!tDQJp?ND+~$Os&D=bSU>hMj7SwXDl|-qn?;kaU^m*djnex9{XUNkoZru?5}qDjJak8?lg=;C1Bx7- zhT&uZu6F1drYT%@OIP+;z@XdTyMnq=YTC|Jm&S+uYehV*d0}m0QL3HdriTHaT}1&v zm)<>_Mkxh86+#=HVZIxg#dq*5`{x*RqX^K7^6E|&xZ)f}jvyY<#~|z!i%v{tGXW&m z3Ei1`vK{MxyVg|bxK9z+y*xz_zaZl_`!G)=^#VXTfq*}WIvr#3>S5fPfzTIHOu}Pg zi-|h7^DY0pDl$>0FpY9vK(wJQ2~N3m{V!jIJV1rv9-QJng64Gsl(I=*Hjw?wio2N& z-RupSTA>#bmnbhR7=ydv4>0u{nTVFU@T=tNC=d zoL4K;>WxTM+uP76So*JQg+CCqs_M!}Gz|ry!Mw5);p)h>t*}F@h%cEG>=qOV8?cDw zbF4Mz{OyCGKjM>Yf{oAS7UM{EU59P6Cuh0u-G0%}|M5=_akKpu1mXEWtNkjAK`k+?tU6^ltlJ>HNe1Y zd72~>IpL&O<4VGIH=_f!V=?(>c0NUw4EKBq&mfdJ%}H^bHA!)#J0(hSKu_Du%l)yQ zmm}XYGHXw?VfLs=w7Ivf$n7%Fg{Q1q zo|)ZLaIzV=JVAnWu<_Q84rVP{tAOVTL>jLKzsY*+MqK7n^L_g#zD#R&KDKUR@;VA< zez;qKEj39P=Q6$#ub24Ttd2Jk2FnJdqkQ3m_;JJ#+JL1C{E6Y%Nm@xTwb_wwwSgc- z3q`p6VIrk~(S(m8vjjXAua6}Xht3#7%0!v_qLBHBGY2W%{*VtEbwVXsZmn3mb#z+) z1&T!I=%?J%Fqvqw30*zCQ&Cpvd?Mnxo;f_R_A!>SD!O=HNgNG89i0rFIXnC_@b-(a5L^LN%)QovIk+@(@u~ zH*0skMpH9g4Tj_pheVI(fIDIJAEyhbOM*&+&kJ4rYK81J_XR$mtJc!OF;?26b?gG7 z2JbOcf)03~rYqZ4K^SgyN0;5Ue0+G)`EAng>&lXks-H1sN&=ny$}j4ElRv*LbjE6Q z37z!N?MroIC)?U8J2rjP@f5tgL$+S+)T)vS!2_Rk6(fijNweY|r?f>f?i5PCuQFpO z?MP5X>sm+pGJVvRf9cY*{ev;r!8w%&>%YLD&ZA2exwu{^i z-fay6!hP$W-<)V14xJG77wJQRNLh?j)YuGPaLOAIe%u)yPz^U6Tm(K*s_ zeu=JK?6MuA*_1=JM;UfwIM+Cc&)MN?|sQewWkQx-*U0XgJ`e9jl7c| zY`@x-sBsZNJqR9|EJb0aT5Q zRMSl^5R%8v9WJH?f%Oun8V=s|1$adImWgUjlf|mds0vQk^?>z+(5Bry0te-d>-^iE zt=Dt1&yq#l^lRa%lIA6Y$MJnq+S`IrI&3xz5g(RHT^*eK%dubnu5_f3ZlKB;+kd~} zr>C);f3*cVb$*?l>-~@Mrkd|+ zV0ufLFt{%;WpZr%&v+XLOa}Z_K9hETW#=;auj$sYvIz1n2LJVnD_OlHVmFVK*auPl zY9*FL4ASNK0Ala2ldVfEqza&B@)G`Mxlwmvg;;}{SnqVX>mMBL=Bk|L(> zWGR*N{B3KGAzl!Ba#KsJiPSwM<4<9?JJ>(>hyyu>%2LE6<%ZIOu%o z2<2R*>i%`uybB#~KmH*0LbnoIHF1V0=&Ar#HU>yO(*fU14p;DKw51~M7-cEdL{fm_e4i=NrS`M`T3uiU^`!C#`3t(hJ%l$E(o>u26U~NN1h?|34!49nzqxZI0^9~A* zN;aq8yc6MFdQaHWYm_N1`7fxPR9VmoY}wDr$NMcLRmYKn%91$G%$`h7%{Mgn2NKT2 z-3LZc_2N6$`+ej`T|0;+`&3T;&kF|c$H>7Nhws-V1Ivf+&)N6m+l|0SkK@fv?#~H> z>rwGLd(1jIJ!xm<&iCEb#>xu9eBG`ho9~5pesaY?N@oH`6Qj`k?hP(lM+>>v)Y-Zhy|o zoFx{bN8jX$Zi+bkL*&bg$7M7n;y{uG_ayV)KG{#(B(^T&MU8@n8+yAkKOdMGItP@` zXh9$HnI=l0VnJ^3c_5#Z~7{Fz=F8O0z^Kp_**^L;)slzO&M3LDV#b}Z9L zsOx?A&qWh>J3hncasOO8aQZ8mxUg^H905z6pZND|)(9d;c=T)MaYtM5A>wC$Xn~;{ z>sfyo>j;iYXMf+b!Cl0|I|#IiJ!XjvIS4C|r6t?n6APsGN*z=+BTX+286v~Y&bv<3%rk+VGi@# zTAsoOOnWBs^fSqJpVw#_3e@7;UQju6c%Z-nv%r;TlIbV2c2I1Yy-SYTk)O1J82==R3L`;Nt8x^)(<^l^N6GU^2r*c=#DG;{%fT{y*Qd9NCkWu zRSd)eRTx0q$cIV+*Rem%2_Jf^OqTAU-w4=w1j)!Yf*QrD8uC}l6%(KXmQ`keRz^%k1)(Uvd>gzFSIubdXT|}6~3@d$j}*fgKU8LpQe#6ePIh~5g=>lL|NyF>-t#s$-qDQgdkSaV4a$J6F#5MYSn&jMg8~z&<%ZnclNHUZkrqkU2IYC z+C#+5A2jI%=p5re#q4V_T@YOeC+0msAL_T0_XH1x!f0-7nZeh(Nu9k|K&>#VpC$)2%__%lsW+ITBdqtS)SmX# zx*dlRr4kdZ$;pzJ&Xj|fn>6i>%Wbq;Nj3KuuU_FN*UBnAliDABhrVDVIj-aaW8#sl zba#I;xh4fNPWfes+G!jd%M3ov9Pm~d9*Nqtp@YhS_KNb8Aj-82T+(CsL|}5QK1O%; zvXqwNQG-u)JvQpO#F1EEV?$(Rc~Z17mDdo$2CW29qThlMZ!VL^iv(}89gPME@ynaL z{ML3t1h>2uR@|CA%Y}0t)tl?}$Von*w?_2hsLScB3osBEKr9zA$9{2n=s?V`qZKuj3x(O#mO!GAerE;h6X+D8pLb+K(6Ja2lT_f-)X7AV zQq`HTE*amv#z^Ra@N|m+mG^0f;>W@XGk$qNnhoWgw9h>*-GO8sP%A8s`RyVXHRzDr zu6)eETFP6Nt%jR=Y$(WHopFQjs2u?LM|tWw4Dw`*` zUL}y%Oa4~m8_5^S{~!|1kB1JT#K(fcL&AfJ77r8dAnJ`8_QMMO6^syt8GIdx9Xw1Z zA@;(Ftq;SuIzCh=mp?!xpYNS7pRYKoJg04w8oGw;JzQuTVAX z9bYx;##)(%pLyiyBGDw=Ve=72g8r7bq~~?!U($S#b>7;pq2HzyVKKx|hUkrEv8Th< zFOBgz%bMoYCxwya)K|uS7LZ7PYMDrnIY+KN`*8K&Xy4uMcD=R@DGaM7?~4F?WTKKQ zGz^q9#n{@d3bFj;Y7w^#bkp-5i~9NDX@hX3X|HcLjy~#dea6qeCgjSN#Mo^g@tMVH=rkqlGpDrjH5>~k*xbO;s` z0^FTXLfiJ`Nb;;RI|cDNrb0I1l@jHp)`^A({fBaSpR=J;UdAj!y`N{D>6&28Z?b1+ z@Y5fSIt9cHg#{wd7@@rMRwQ~uOsZ6`8#^+@?0#IPta^x%7R%HwshzZqe`4(2@x7OH zFItRADvLp#hOP2kO?!W-Z!geHGlzcG8gt6I?GV0yNJOb+!|k{#OV#_cC!4@}y(+5^6S|nWCYR5Hex!i`eQYLw``Tq>qiD`=^Qr6g3 ziT`SW!+(P1wQ?2<{JWL}qPpCqxKZ{t;y?;qraCH1u)HDkF$GtmF0%<=jeDRdVIKiH zwgs-Bobct{+{)EUYj`b1!ZZ+7H9F^;WiJ7_7H?*X(g|>|#5nQ9H4Sk&ek)?I0LIjW z6G~^iDuh4F0Q^&KNjjALZ8BqKQsb)$iS<+faP1=62-~_W&BmtV`7iR#10uqL>%9!V z8~5hwo+Z(`+`CCEv;DuMXKLBzw?9yIrgC|@>PYAHi6JY=sa@~$di3xoBq=D=ocrX@@?VmTR4AO#R( zuGZ2AuTPC^dgTdG=P!wnhK+?dR!==}EhCIfve09)4j=ruhwu#G%lfUH8=9T%n`8e9O5YWgV;`@ccz^Ho!O@icb>X7*qMu<6vfa-+{j3TayLP#36 zL1$oTM=cW<7$KmVo(zols;LLe%fY1ALk>kX=l)iZrdCNgfYu>}6ot>VzgV+d#Pmp? zo#Oebr>;++4X=6H3Hjcp8WCi@tzF2dLJ0kkZ#pw8gDtz6_a4zbpU{J)x<=AY$ zSf6=`vGBu#^YwR^m`m1yWmm@~RaGkv<&H`^_ z`uWT}6ux27u9{YQr=R*&txsXkE7w7Vyv}XK+xsM3{}x;6u9|k=wOyC~m}`(m3^rG) zZR<{z!aW)XULZA^qTo^#0CuXfkMipu3^SVl9~hBP>Woo5`Pc4{M+mM=S7Yf3~`+1^g$4+>6cK$k9f z!!%lV@(#|J1~`*Vv;IFAm`@Xyt^cLS>xE!xt(iXKQo+P4d-Srfk8@Sd0V?XgJ2ugT zx$@0@ZtjA<3)Ru-P65k_%_b3MoD_qc z8X~t?XF!;of%!>4wG3N{fiZyA4$0Gw+>ChA>8LW~M;$LVG~)J@x~Fqco5DKC^r>N| zrX5vzH+f*`!e0mL(`qY95F+|evOjl!S{Nx9(ds*HC@~kSae#!% z_fsaNT?St9GOcjm1YnLibJmJncd`*YDGdf$m-*dlf#6psf7!1Av6pl0SZ z?PAv>Ax7!k5S)^y(&$0ed)E?b9JGSM;9s9ekwvezEIqvR8>s!v*Zb1*+$Woog!V6| zmH_gXg2)(Qog6@4j!yKF(n~B&sugiF^1syp|6hdVdosBH|2M)iz;#sG`dj|#e!S5FVq+FU;R(%xJ}%p2tof5T?l591k39~m!cxlGA4O%=8I{fk zSl&83K}+`?hdYXH%wn4_^`2ud%{b-RWlGpSTXX4oJr~d^=0e*P**Q3Ufwl&l+atDa zGBY8l(ReZOUMFLV_9Jz_F{y>s_4NMtOhV2rvtp*lCy!o*Na=#8M)a(^U%;QlyW9K4 za=RKu9hWHhE_EhfedX+&cQ$gJ%R19#`~8Y*fJtNOm;Nm8t6SXyU>=H}^k7QVk? zbyeJx{A|;W)0f{!X?PJ!(Jqy!?aK8dl&T$pKA(#3C$>;j!^1hgJ6@%pznnthbsI1Y z^W?Goxn4 zi&*1&x9VjS{9)wN?e$f^)BbTSAV3&wTc9HaDlLA}E8wIXf6l?HLwS%8z4O}BNd&dC zaIFi~;M{GrwE!hQ8PHfbMnY;few!r-DY`>D9yj`7r-xHtBV%+2Wn~bRfIc0-NWxpos zPXig@2MLlPxM9@3u{fXLOhNv6yF7xWaLzXCR+pyZg7ymCrD!Hk=!Nhjdz43`shj!e zt9+$H#@ZfT@I6QPInuJ8vOWs4?BmJ1?`pVdTH+_P6642u ze#dr)r^Y8xIpl>AzR{we3rRhV8F%JsnBL7d1)KUG1}3b$RY`Jeo*FC!L6{^hS9M^D zA{KEw5VYX?mXzTJHi2&NMmWoc1OKFLb=M{Q*rwy2ZC7hxU)`_p zYIvfPJ2*EI{B=c$bHywNeOJ|U(aFH$-jNKk1M5?Yx@x={`dvP5M|e|;5|a_`hmuj# zShR_D5a=1W7EwKdj=BPSRe8Ba?`oGMGAGEm2u|inRgYZ1)7HER`4~K9T>Pn8iqEj9 zVp`IoKbZ4DTiT>tRtZAQk{Dn3ak|tKqY=FN>1<@Q@8X?iF^{M-p$@NTr7|ouMXob9=yBG-yFLxu<#1 z&#Ml)@84G(9~Q7M748!XlhEJeaiX+T27`?=whutbA~Pe({9ZGO-OtrfVO#r@Gc>->EJmK1~-eeRH*wgG$KW z(jF8y0QVG+EwsJR>Kf)76yK(T+9FJMy5h@OJVwr{jQmPzvfZZ)HggI@+4!{TkrYs< z%p)WwlC|xa=4OLr${CE>G|qH1oIy-{sNe8W%3hY$IUz>a6s;vsI;~g8<=$G&E6hN4cj%+ zep%zlT5U+%s-3$otqWvsDb@2!_k=`MX?F+$J;nXx*W_t#f2s~b#zc*nB!P82$|1Ur zWYgToH8CLpVGL2|B^Gaxm>M5tuCGW=_Tn{AVDreeQJIhw|5YFz6CYd<7!#ET?eI`` zS~%><}T;`x_;*QYwfuPzm2~wj?%J&GyRj{)vp7gZ3``yF_I+ z6JlJL#xp~WT^hUeGe(LGYI;iw0i2mZIJY~W@9)lD9N8S(MK>LCK(j*UIw3J_B2De{vCPMjs4 z4JkVo-&o*gn)bsI1*{l~XUE9|hVNAuF zm#pP}ET3XYN>}I4OlnxL(LRf!0Tgv1vZ|<}<&TV<1MmJiyx?TwSxK*sed*d-StyIfpEnnb$Q3svr za?JKoA3lW&GVonTb^2$P+F~$}YBxHgcVdsNFf(yfj%NJ(2g?*?8ZL;f1}~kvJFXT| ztw?z7M;z7y@#)>hzYoiI70|+sFu&OpI)K3D6h@#yyDQT;v~QUCZ09&bwOfJba&I{A zN#Jl__+Ubm{Olfu>`_b^J&d_icv!-t+6lvQUy)!kcF+1E@mzm~2T<+wI(^q@mazLHl zGVo{Xqtei3x!t{x7=d+o&Z(SaR(!0%>?E3931+^!h~(8olgZoFT^}y^y>VG-q7hJ# zFxL}XaC;=C^1i8MmQ+34;hKu!5Hm|486YDyt34aFicM`gG6K`p?r~>IxenQton|Jl z`OH(YMTZhu5i4o4A0- zQC{ACP4+wsdh2a0qEKM z?qPt{`R^Ef#C15X!R6ZrCBlxMfa7o0zXO1)-GrZdAM2Y(==dSrsqaQe*GrpjH#rSC zc<=JyVc5G~8?2WG6l4r#E^gs+3f^)KS}I)P=x6K{r_oVwS3vj8Na`j2?YX{XVTJs< z6si+pAqVLOTz-HG4saJ93|enEwY@6W+Qq{pR=>){EPkEiFM6(=q5 zZXz(c-F-3veSjoFGZ+Dh?|4#op=10;}~R z^#UR*P-8tvVQPS7hm&OfqQYblL9;^b169NP$I@M11zvp@iJO31lpozq_ zmJEGNATr({=ZBF5B5eSN0Amk=3tS@6;=yoP4KjmS-@PEP<6@YogCm7N{vy!h9PZ(z zm*1}=;<+OVI{1vPpZf(F0~Ldag3HmhnimZ^q8SZ6$9btx5crM26@_?OhND*fL+-U%wwLpw9&X^x-B_5m?VqxThcWImGh{^R)R!gps|ChW8@OuVx^~VF<$h3Z?r9 zq$#8v3uZF{ZVhH`Wnvf<_zzzGAVwye$w4#b2_a1On$0_nOEp79ceB%A2DBv=g*fyy z-m9B$ZZ$o1PYtbP1L(fyUsRnu3>rrKfJ*;C4u@S(oPv`Xi0m5YzcNF?%dvlaX)p{n z(#?TG=LCM9%-A#eEY#tV=qcT6p%NKC{}G8)>_1RV?oJ4PpVM4WM^Es>j`C>o_5JTS zc@7-|EhwF|hhET6{wP%d@#7?c`H70$;N0B&>lsx;b`tqIVj6mEi_^(Ypq~a%7Tphd zS4dy_|8z66z`cqowNSIruJ`Z>msq5?k^RDc)1(x8hoiHHc&f{9P$x;E@CKmoHG^h0 z!w8(A|2pBNc{rZV%>Rkc>KeBAf}5J+o}}Zw?F7%*Q1=m0cp12q8_`;(sfFk<9FMT^ z9sc%ysa@=S{y#~nsFk~ip6TjeJx^N~HPO{4fyZMT6r426ui+c}|Ls_2{69LD+td7_ z=?u}hg-nXUeo;$_qYQyBb_Tq=KxKBV2f_M*_Ht)gZnx&PDmt3Ah1Z6gJ)CBvA7b>+;1 z(9QuyGlD}Jh7V1xae!Lm^|=ho@Rx9R%R6LQ8vTCl>@0$MB=dGmudfeSr7zo{(=31C6T2|^bU5tv!+G&l;)SgO{K|0x&$65 zbOi27%tBfkQ756s)4kMDB%|*gQ^k-EezF0>i9@ULePD+p3>>gCN_uH)VyL7WG{p?GF}xMaZr2tGyid7odV7 zKfMldG3E4+k@#JXUUoQaf+dp_IIhI3Jx9#0xy)XB23gbP@&;_mTm@m>{6fYhpsIE9 z#m&}a{b17aAEB!IcH6{cB2RGfA)ILzI*6@wp^>+<N*J~Q$x+__1zvJrM} zpUk)7dfp$mIvWU3T|dzKkOtm^EnZk8pPLK^q&NuQl)#6q!VkBXOFXP0|p4jlTKrWFTAbMbxM{AmUyxesgwqAtS?yBcF1 z**;u?ZuLu+PJ^fozV<6JAJMGPeTphKsdEQzN|U4BUgDpM>=f=%ey!;~H|1QW$6CQ{ z*=Ogo&0mRps8+jpjbf0YYLh(%m*8lS=+ybJdl`#&{eYqft^5x`Gte`?t$4p_)J)@r zkLgAHs^$kmHw2%S5RYOw=8#?#-8$Z@x`+H9&kY-^?!|{I7XoX)`yR#~BOT~r9j1v4 z@n{yes4R@4FF9|$!T&B2sURI}Z#wmBfR+-l&N=TH6w7R*nc#qpwr~hOL5HIaU^fgO zcp4LI7WL@iNH{=gOO@b7RCzFE3TyFzP9bdw@7zT?ejGc+6KzaOsi09ZJrg4;^jhPxAkS z7mo~i64G|URX#u2^YKC$dy!=9gi&y3z{KwTL>X@OerIflzOI3ZG4~?DHU4{wTOQ8; z4vGt*{&(a*eILB!GRG(o#YjdXL@da!?FhUwFn)(J1)9%Ipu%&n)ZDc}VkE2DUi50% zT(}#Kh9jDK%ng$3cae{3;vCZqaW8_dRt z&ny3L>Yra$Y#cswz^6rz#Pr7PLchllD3IU$?1%zV9m`pscun+cIC@4)l<%JlD6ul# zI?JhduDjgqrtI2HlvrV@EJtJ;x@loELB!wR!YZGEWJ#l|Y2fJ6E2}E3JDk3C1~Q;K z1AFzwmS0c_NqX)fKSyfF!j!6#u51*c^mccgwMJx3rO;@O*MtWr7>%wwuEfb6J9T-ID$!6 zdN$JGPK_3^qHf%_`r+k0P4rW*{iL?Kly5#qA%i3xih&6D)LXu;s?oY>Ji@tNdh-?7 z?HCTdw9>o|$}!2>%J0_GTEI#^ZOgK}?7&c%;Ou@suf5B<0VdOeB8Hh%0DTJXqQelT z&kwD$7B!Fk?>*bHSt>X2nm(8TzS+v6(I|6z@*JAwBWSHcy^_NFZ}6)j73# zhZ0og%o9|Y%vrvJlRJ`Gi)|HdO@A{q;0C)zFWhM_Fc}(C$$v!ng3BLPN!dDni}llc zhUUT3HM{dfyejQ^=0MqM>960pc$#>SEutvZabQY~E1ptA``#DU2SCKDcrrg}Ua#pn$`thG;N-5WrDGEG1RgJ~xCfU25 zt`RO3w30YAsmy*GUm#{Z)jgP48kw~A9tN6G(2Y?(0PI1qw^uiP_$sO;uM8h0x$N<5 zFOD{8c5;~IN8a!F<^Kd8iIjy(jLp7;yO$V&F`L59vSKMbOr zx)`Bf50998;4bz;NCD?6CR&7KD`rxyVs6NDkDCdk^i-SPljH0+eeEGE-)<9% zoN~K)!Oa5R!pGOiRRqLGsunbC&6G&rzDj+*2S2}4Gb!{LAWv&p+M7J;X#0eOzJuvg zxC=HxK^wujgf+8OY5U3D?k$rxF8fPtQY@7cPn3x{wzbi)PDB1Np%Xt8$h2&h53}45 zlngL@B${&4EZK~&DOd2yaegRj!6n|$?{O_3n71)x!jhNHDt0*&XeFRtV$Rpxm*fJE zU~5{7K8fTaV;)R)w*c5o{_V8jaXzH)x>al!S@64s!P{7*#0=_EEbbbKkh>{$A*no% zn!vGPa>7c{*6apKzbv4v=V>k>7;o)Xj=vcLu(!wr1W(gO~+SN2X z^K8=8YO0yhW`zV8UepZRH&ez(oz`;0gL?iV60;g>z-|RL{PcE;ix_(60LS>Cp)N3X_9yuS`8`8JI2^KQv!zz@&mmH{^ ziWR!=g7@*PQL1)i-m)N+1FYq#fvg#}CT!(> ziD51fXGx3Bbr3b!^x5cgh1+0PdHM>4PE8!*rEDStWzSL3WTAl-cM zFO++P$zf>Hwo??iO%eFqS#VjtgdAVqgB;nHBgM~`MBhvEy!Sg6h(i(@Or=PGJJ%VY zS8xkC&KG3mIQ?t63QkV?ftDFn+?pRnLKkp^-qIXt3}y0XF0rtb(7@!8TTE(nHaje+ zT1JTSN|D52aBU6?t9ClF#%#l91NIh*qd?;|WS9{0E-Hd-c_I=DBDXt57&A9(m0Z48 zc_1BeJHG^(aAx~@pv;9!++mN#xIIWXm$tKQ1M`^1G8~VH5U2b}!x|8vC3?RWY9Hop z+(S>KN3JF{q0(;R1}<}Zd)Kjh%$Ye^s81Qm0sZ2O$7?>62n#)TC) zLo2G^SfG!2@hw7oVnXySEuh~_jA(}UXAxTspIEHrN_VpsvWu~Ib&6@n36ReeaC;AJ zrdjl4_~AKZ3D<8cB&n(!Q6Cq5>8vV5u{xm^e(BkEY2WqeP(6exJ3|XG?-}X%4kciN zUroBWwR;xv00?o8ZF(^@>=Uh@u+OV`^tH%CjXv)l9F9)?QI@25rec1-J6;im5Y3V4 zp{7Jk)3CK0NpGykUU?#l&{}a9dGl+ls+C!xf#FInEZS!H@R-y}(i}kk#{I4Q60P0Z z_5E##Ima}{dJN^kaVkMy%2MuD?ObEXvI`aC<=5H<^hA$|`=kaNsiWr~xBk;WH8-5o zuQ0Y)0cpi=%*;>GM9N(=LvQ4l>EtJihmq*qN&19g$brjGQ(=GaKZO$xQ6{9tW5;EJ zzh$ca)HP-pE4pS5jOBCmO6QXtRnLS=EX>H?FH@q9TX^Ms-09}1ZpX2YVfROHOD5=> zF~Z6O6KzqG_sj7FkI?U%|JxL_W`v1t@o%VX%?Nv88~ERmdHkOA{nNB*WitXBF`-Di zMT5IOQkgcj`GCL$gJIL#yQwVN+4y0Q4O)LT#~OAF^m)M)Y~{1<4+cRaMt?|g4!_@7 zu&A$YF`@UmaCN)gZ+~<6Q)qU6-Ft7dCQ$MHnlHEqF&n=TK0A`zWLI#v1li*p84-_NwbJ~~Gt6Y;3o`?Z0Vlqg=~?&KWBK-kkJhf$00U-V%ZJTP;y*1{NkW* zq$_R%?X)iEqF?_G6gT|zGEU)z0nj=&p13F->(kWaZmk}$C>@z4GS?0$9a=5lm5{Z6 zO2+mMK_u6${HNs4X75aDm4gf~znWxG7 zr~hs0WyK0M*!u2^`VRWOhkoE-2mj~#o2(0XX1<2yTylW2%N`|Hz3-_l;5`eU!Z+sc zzWF_OGMfk7`w9*~5|eMYw7{e1+Y*}v!Ub8Sno?{6MynKA!PFr$t1V3f&s zI{!BwX|eCF`#0LOIWzukIJflj#@8Oq#42|=nmVi+qA>+e%2xxGwbW(cK2mrZOoP~ zryHxeQ^ckwR2vSL)B%5pj~xUwm0kb!_ve8rUuNga+QQ1%QO=r-qFyp`&AOaqMpT~a zY4WXtNk%xy;}u)KKaV%LjXVi+A-MqOcFw<-Lt0<(-a2^(tUQStXPS2rF7uCZAYJ)Wg&AcjmT3QxB;U ztLt&(wgKzNz=PQB1hTP&RMfMP;wLFfSBqdjaxt|>tw6RAzl~!JwA>**1(I9FQSGKk za#~uC?mnuv@lkBfB6Wp^8LdKyT%4`6nRu6aI5M%8M-^bW7f~-SZ-7|E_V4#_Y5s8@ z@(BCa-$y-q*p2=+=K=bUTwx8|T-p{Lqu^`k6S~MtX!GfBO!?_iLr(-sg1n|@y7A2$ z0yf*&Jt ze%$ddNvV<5`1-+dh|m{t#>9OqybMS$Ds_PUdaKg8T~e#s^Z$7XrW{U>NJ7LEODB83 zQwkor_EU@?ev59yOH=_$TyMlMTEN-c8q0wA*y-^Vc>_D&m)%167B+?9@uJf$m%MuQUp+nh8OOvUT5A;`jwl*lZ3%;D9y2f66?`>gnxp z4}7u+6th`MDM1{%8iH-=;jkppJ6B9_tT=mK4rZPz>n%?UK6M0j z9*(4mOC1A1bTRR4BSM-^OQ>c)3FonnY1?gDi4v8c&tM!}c#@z9?+pP#028}%r1x_@ z9KGpehm?PCJ7WbSH|o*CPCwK;I0d6hid$hM5uOmWKqN>xN`Kq$^Wcz~9vc6=$$C>;Wd#Mwv0ZE0^+o*K(4D01MP*YAmq>i!gd zGH(b2BvzkX(+hO&U=qK2@o4s}4MkQvQ=wir7)AM4r9LT?zGE1-bV*MZ11H!qy~UVk ztOvXX?V7B&92KmpmU!p;k*vWyC1srHlJbKrH7&L*XvATbt1x}HR-YJAWR}8zjnHjw zq8CdbkfLtJVB9j>;*-Oc=b{%eEJ3Dk$!-b__8kKRzI>wiNyD)Pm|XMPX|rL4Sl&|S zZeH-HbaC{=8!94s4+q0r8SUZGHNuu*LePc~tlC&Ykw$vq&C9mjXdVvN{z%xH5j9ya z@g=X5ffn7_ce47?#!LSk<1hxM0`~CvA8G4OW(s-uoQe#=yUoOx_Wj5;EhdE|_L36C zACj!Y)E%@m<*kf`%zY;_j3_nfsbV{b#28PWtz~%^)x`QN)%g_(ZYhse;f0VgVeS%1 z^?u2{)EvTc4Uo7MuYgWLd9O;szGv@mz{4MSbcKcIiLw;hbw2l9KiEtsHPmQXn8`H; z2f>=>o*xm^JqIyFlx8Xv%cwAO*{wJHH#s&mak15{>Ry`92)-P)=8l4? zCu|C*kEK2J4t6CZsDF|IyZYKyx-LWkD8m|Ir{w6l8ipi$BdVo;A`s@lSNZO>?689b z^21hCPtK2orf|bAx+mW+9y`-uWAwYUWwYX=Yvz3FB8j9sa}x{B*?B+vS7{=0m6^Y2 zIAxGM*f<-{cV9q(^}NrQ@nr?i&~3Zuwn>?}ON79SzBl)k0|!(#^y(u4U>R-~YSz2H zx<9M1HNLUAjqLgMg~0*O!?Ne~_8)9Ud%7SQ8zy&VM63sen<IZhFq9X24LR%8bIo6mm?2 zM@$4zdw54yOZ2H@S2*H#Uvw?6L+CO7^w;qEo$k4pu~zcJgyM88XT?KzTeR1QhQMY1 zgU`BZJy*^bzhk_d=wuMGrEMXbx7cQTh>Sky_k%h7GVNmMP%QO1y%J zM@aYU@nmo4Z3WtY3>q6pJF;%M$XPT;D%_vuX0iCIx_?&G%OHuG*3i|2wUF&@ zWMJ5Z<>Z{KrTi;653Xv4Z|`X_i3;I&rE*I~$}2)(TF1pesKvdRjH_}k!y}6VXY{Q# zSMr|Q(rA6FnMF>5y`(8G*94VPKP0ubMjyYYlhUcTb}RS$FdTSrWC9u!k0p#Rwi791 zF@{tt0!fK4>?L!L(!OqGp0xj{LJaqN*kI;E{F?(D5Sc zwyj0HN2j_CMt%RD=*|oPf9c#pVoQ@I>Cnwswu4vmQVEW|aGG)tCes`%nq&iGyjXI| z;ittHKKq0|i*|42O_qKwC{Y4lj@Xr_O+;TgWp&KAvr^1+3+}k1+dyz2lE^J_fA~9n z=+G?us2PvWTF^SMvxSepgBE}O1V4WGf!`oMY1J-BrCnFAdy(+!j)2Pdn{}KNo7+ow z&O&l07Sz}OG1Jq4f;PJ#-&o|^wHaT>Icsg~(OWH#J46u*#=*S&;PiW?4z{3(_7sAP z+Q^OQ*8zJTwX#bsS_*!}^Wx#si=?eLo?2*t^}$2&6D$wv{;}dp1@%_QfnlU#Wk5HDvxMV%l=$`5b-uBHk@58vdlPIjUU7$gAH1WW7!I`v%;x{g** z4y8ly5xt)=wnlDkIF-vXe^lI)ZBCtLNu5xz>&%qnS*ZP+$=~AEr9mjqa;>^jK7*AE zwVB@I^DYCW^QO;k^6mwTB+#YfrK*hR4q(kRlre?;Msz*K)Hk+qe316Ya%Y>_LK2D@ z53z*&JPb3IyxbP z9XX-sJQC+w2ffoDqa@lHx>{J==>E*86esZF%3;GD{E${-SGPG{Ot-Rukx@d{T*Jjh zfZwkwvPsgiD8vVs$(_5z1`dy;qTfGdIJkIK%%W@|VMVrq>0N$MYKd+~|I9^}5>v1_ z;~{|dG?ib&x&fjqJHeLevu&ccf?nCO3~2Kt+ds2sjc&Mb#f7O`j1Qja!|Zwm5d4B$ z%LFs{gm)Y)7vDi*vp>U|Qq~d~_9Aw@S;JVObH~;v;j!e_Hf65hx+JunkrfRJ0OP%# zBRqbR^EsiPt_Po(6jljP+yllAL}ALA zhk@3$j;zCGbBk0ngmw6)6teR%?L@AGCIa*77s2dhP#3YF#y_$9xwp#fz2k%t(g!Hg z(+66chOBZpN6v>j&byB}#w(FPQ0>z5Y`0RB-o#|)42%#><``seWjm;=+49p^JpO)u zx3WtP$uJt%wp(|=TG7m%7>zpba6)dn!c}^H>XKT>M%m=niH?R;b2Kbp6r|$&KOx=0(ClPx2792CZ*V$rO9`*RJEMZP4 z-HfK)i4RQ2XoL7wiI%g{^v#_i_u=83L~%;-AEI+trL*DM%kCat zGk1&4P0k`oe{_>U-KK(})&1d@K70LJ>^1l*7}nvpgoVh95=V*qJN>4rpmkD@IGf)r z3x1G#~rCz?NUABo&}$9y{5qUB5(MX*&^S~4I8y8;W0QkUzI&x zEAGvnsr~g9N4K#adFJQsE!5Y>qiqg4ghbzRuij5sp7yICSV31a2p>(7|7|`y3eGRG9l-en~7Y`rrw%yFw;tus~by+X{ zg~8;iIbVnD5S`r}^6_5TokQ}*fkPX217^zf1laPrW7g*G9YOYeSfKtOmq^z2eaYDJ zd3d-r`zo)+B*Te}h1Pib`tt|zcK-G1jgmsUMtpoZBZf;UU_mk#4VPGzbm4c+$1^R4 zr{Zfk=Y&kJWk!p0JvHKFAOHdXWPk^68|pwwa#Tg0#MeA~1HbX25dWQLuwQX_JOU@y zn_XO#K8P9$O9f-Y)s=u_kJ1MPS9hi44Fvc61BLvjAj_Zdh+HhM`FrTJX`=uIc!K&g z2P?yy*hnufr< zHQr{ua3}-U--aH2G>IioD-2`mDI(+{>9i z703I=RjGf^4+%6393~_0H=9rkn`BS9t${<2o26;J*!xz+2o2+ftUj~h1HTcSS?UUY zjY{-WzMt|QkC^M#7L>LYkDQ6=+YDOmJGL>i4Y+B{5^RO( zgUYaNC5E)+a|drehIBaa26#T38mIQdM!~<9cq4+$d0b#T`;6y8cY-YhSvz^!{(}Cb zC-D2omr%btdlP*RxAU@|*(rK(dAJ;Ie;^-pGt@X+=DzQ^K<|KmPG~OK@~8ujG;0Ra z)7>-DzOv_oqr4e<(^5JZ{^S~oo-8uwi?CzssMCpku+3xb7lNei;LhBSuuy^*Ci+9KkY^dn{@mkh zDJdaTOjlL43L~VZ0&@fo@4D+W@(yT8nZ_dZ?AOR>B6B1G(eSJ61IP3b?-x#_9RUx*CGmNdhP|@)Y_~nSij%oNc z($`|K;$z`)_SGMgTp=%oe8v|^+)_pTfBs>YrjOup)M0=aQ{7)~90X;O+(X81D-mLc#^Ba-$bYFWAAPTDR{5=j*w%r7pAqBi$m4mO?G zHwRnKBk;O{f&-aNszpL|aI}5gu4O;zAExSU9^%~7K@?9RN1MJO`WK()*D5YsG#OO? z$4a;VP8i%(m;Ot=_8iX)4$fOS*;)}rdu5Z6Ecev7Fl;<`S{OQt98JP#jWh^#eUzJ)03`WCs;l)4Lz}|)<1?uF%s)bjvWv1hj8-6JCpWY6 z7w3(MVQN?OO7QWRmm=&qX<}#Idyq8$-9pf|`EAPf^;GMMQS5{mOU`UJsXe85$4AGf zyh3qBoQ;o9TpB|$tQtP-Edw1$vEqOZm>;Zrb1_E7aeK#&1@E;wZ^VP=#=?(t{tl)c3RAhPJyznMP{hZ%V z-@YWT{Kh|x92rb$vDk_>P?0q1C@Dtdjlopfv7?bmBOZH2tU0t@7vCY|XJpj4+Kzuu z9Nuyscx6*G9s(t50i|k5t5@m_uV4!oLj-#D9$#hbuzqwi@+g!~(C!a3#dK+E9Jz_$ zg95jC7>&9F)0cZgyO2fwLRUgFrP{Ah9xe6os!+$wDhh{rVfAPEYfGk`iG)O3=1;Hq zcPl(NrdQD5$Za6DOln{^MjXdRKRTHT2TZq~UZ>pWNC7 ze_E}8enBRj$~lg+O9vv^<=xU+W%FE!Fkb9qE`Qc>sAhyJwSs#ax+ilFTl;ILSQBEO zC6>uAPZP;{^%p^uj*8o>Ai!#dx?(t=+v6}C9K#6w20YHVar%y8#6>3CSkuV&w@c9K z4NR8!$>v{M{A=PaU((2@xq%FIh8IJR;%VDCM(kNk68+f zOZYm)webAw6T0P?l-i2U|MsO;_1gD2gqw%>1=aOi9^@|r@u@?ruX z3RNDqfz}Xrd#NDx0}o?FRFs3Rww27URd5J#25rPbLg=i;WgsAZN1HwhR*v=h4T;yU z@yqUrD}tGtFm(ywnk9|H7%keWekYlAvber3YE_Tnjy9~~byEA9YG(PeODlCkfFuvI`%NQs8^$;dI{IDV4lM%E<&Q9NJ7MHg%rakQ3b76jDy6dZB zP&W}!NsTI44tMvw3M5ds-g@-4YKFK&Ho!WVHFUh&y*52VICR$DJV%wJxI#9B;Plu% zUlX9lH=q=;Zt(r$`}+9$BKN%V{klpaU)lD3Kg!^CJAw+x=$~%O8(+)vCnhZ@--Of8KF>-H{YR_>1(Cc0R~-5={R<|H;$oPW2ZZ1Ov`$hFoU_^_$(aR8;Gq1>woW^lS*yJN8lh!bEKD zCaTcnrT!{I3%?cXV4i(dAWee-BIrzU+|l~9I563t!FtH3cKb#++kK!rR;Vv?k?Ki% znJHdEQH!ni*rxq=0`ktv&w&@1bIEWi#iKN$e8Kv(67jhdXO)IX2i$+?CNj{KJnZv) z{JxDT`*Sx_#f!1#Q)e4*-Qm9g}_^bh86-PN2A-X&0cOyD5j z+&F6k4A7hL7v2XP)~~oK*c?bl!3H=g^a@T0%5L}>sdnZqhb(dHp3=?Y3^8vG`rP`^o7H48Pv`66eVo?F#;44s_m0-j7C<08iOtgF2r9ZxKq zm}s(eXJGdAve)JTfu+Vz4U5_;wt{_;D7}K{CNX&80##hcTaF8P^s&w}39~2he|o() z_mW2Wg9D9CNc83H=t>oOKxnX0x4g!S=Y3VV+1E!QzFwWfFclaqcj=gwcSxC53&m?ABV<}&;Q;SU$4Dzpn68@r?l0~tw# zi7q`puL62uvm0C9CHbX9ns7y(@D_Xx8%+Mk7C2C|k&o0sdNeynv4piLHa`AocSrg9 ziVwizE6umHRN#bIK#r|#jV-fdWJNj=stbD8oJ%p;B04_jECaw82C$wniLmPvwTPQ0 zb;1~FinuP2e;aMyb}qC-LkU7N?I=-6dO<~^?)b`xH1S$YsJIrEZ^@3zSA>F?d1T!f z{`->)?e-C7E2%3-Kd(J%>J~StV+cjKCaRWo!92j%7mGGb?)KvA>cAI!ciKXi11Dxr zZ*mVRaNgM#G;J~*`qaVorG&D5%ktsM`TF)^Y~nTbY*axquT*8m$+x}yFyS^G*i zt@)`Ys*;iB${D?o`bUWI;`C1oh4lMsm0#N2L@+YfNZqF3A67rmrxc3X&4Fj&W60f~W&zG*X17QUZqA&)jT+NVv<4Ygg?<*E z|5Z=?@w>%-*8A&G9pJk{83A50FZib(3jdYgL8jsdJ}#jJKI|mG9icTo!yG|ab%a@F zmhjh=$qc-rI?k|N;FSQXP&T!K{G1$0|Kj!~zIzkF-n>^XU#)f9mA#GzQPO>2>AI*^ zQqG{`V@q0D*Yd_<7n~ zLqKfL`@uvIE?X%MW2qfNio9sl{j$(5iZTR3fM^|&mer>!B$(9|M+NGt0=3nt%>JI& ztUY8B2oK zhx%$?wQUuRt9QEsm%#}--wPAO^Rh7q#Rz{e*X!q1<}>|P>V0Rpu*Vb;-Ptn~d&rnq zsuwiR6qeh|o^qFo%~9jyeX*@dy+@%Ls~Mje5IotqH01jDkPD&iBCl|`z9sqza zG0LRZ!4!L!Vz_CxZyzS_lu?7TmgTk9X14?OlXii-lyryDc0$51iBznhG93#%av@Y| zGqaG{AAdopseB+>b`{|VQeulzaHsrb>dJ0@P+9nWa_k+#ops>sn^T#{QJHk&ymjcA zf{41+h{xLpbel4&C4BMu2TpS~&Gxjy$3*F0O{^=dV-~;*W>jVFl zTi?N_ly{z>0@Y3mlq!HtJ*S}zNO8R`)$C+J2do}g)731GEoH=idUog+{8efu+Uk?~ zI2_X`s`_*)7-`utQ{zswTy<6zJP7wuN1Ykm#hO;JXHsmdm+P?JwMMbcNc;nkvFKeR z-(>CGKCngs-DJIK9zpN0zAdy)cs)Hu&w{HbxKC2}dKB+>(h%lQGQCK2ktv<~C8RV> zzERkS!L}kroU+DTnJSJ(UHXcXlSCUP?%8sC(;)+}V*0pNpKY=0sDe{>h0TmqgM4sD2etbd-@N?A zi42q;x!7lclg(G%MMe8u)huboXjGTNa7=yW(5qHa2fNT-BJG4@dlYR7)*3Xc)lsgk zgN}dRetT3}urrP3;~i|n&!#~l1*l_grXN2Je+H~8?q!=U^#q-938aS}VJ-%i!Kq{f z*B|J@y7V2m%tw}EIVAwe>m-B9D}NqubQXTC*QPtxm1jPTeW7=L*v7Fsy4+K!+dkkn z+w)n>S4B{t^rESfZg$DNjO^)bd6P>pGymlS9ONv^Fz_MA(7^mm>r|=g`9#_{#POjoi zIicAA-P+G%Y%Wg^3qMd+@ZBZ|m!6-zPe>uH1Zs z!3-M4L1lv6j^h{MJ7aZMMb}3Ox~=oZUROnxZ+JR z_H?^^Nwv_=ITCnRI&L{yO9k6kOOi5|mAPX)v(EPkjs2U8BZc+iYc1B~*<3BmkImX^ z_|r^d;wJg(MY_Ow-BV1s0&hcbQta1|ir*`Gf4iEgj<;KQG+BFWwPXrhU(`A)H|6Vn zKihT;XE0>iyFvT#K`&kyj`C&psN$%h8`isOUiybD<-Cev!t1Ykl-xhL)oTeBN`4FHoAtwLbHC(3#ikm@IvoQNa!In)|});kuQvSNjz)}cSrw#`Uq!s zWw|vSsq}itbnj1AxRfUtR;x4W!;;hY*YaNtW?L8STmVC%xlITHL{DlT=JF4gP+}ih zWi%g#Sl=R#@Cn`m!qT&?wwxVb^(E!b+9S3!?W?V7{3*f7!^Xi{HnK?*(vAYFyo**e zeIsDOu936nuDtgHL{D+`VO0s((G~>irTvA4Ps_K-&W=HyOY=(gK(e^rH`f)7%9cM_ z!Ivyo=XNkiu@_vLiVSb%pE@Tc72V@hk@%oN4rYPo+SfU74-2mBTXajYOGop-p&8ndRROARXl1cOp9}p=eB#)otk-&hMa{=AK7l0?RCvHEM!pC%^1RD00l{SH zn5+lY7QYV$37nw)t6@}QU2rTeg#?moL$Nt6Bkib^?IcYS-T)3U3jNIfjJ(NL#!gr32IuK(E2B~2`sFQf^NN@}oZH_l&>YQ4k27D+N#gHP z57jPzkT*^RNw!KW&I4#N>WFz0XXP;{Hu&Adz9?`s4{f^FN!--&9aHd(#o2A>^S>7GMm973D zYD0J8uP#Fx*;?e-t?CC?QJ1lu%+i#zd3Y#&2X*tNFg{#ZOS%WRIMS%*$8V$S&JW`Y9>ZQ{2YSIV#z4tt1r8(`*F zLy(1BRg{MU7%=98y{qxrhsbhdl_<&ZsEizebh}jC)7TIm#Ck2N=a9*<`w^)cUcMU3 zM<)i$!BzaOA%2hB<35&~}8a}K%MAgU`4M!Q5wPtPdw$8m8kXhZ9-yi}8+e>Ng zx!QvX8~3hOf7g6=nI5(3l$hEVF`-hb(2jS_VAAz>rQuV@$5?+PZA4TEl88oB=sxLG zqN}o|Yfcw*aP~An+BYe)M6&?y*+_2}gz7jMNkx6XB<08gDmORvQgi!pf@3y(tcH4_B z5BjebD5fu23<*|Yz6M|J7Cv?Lh|NGu3f0aj&KG-GF6*a*e6YMJ$4W65_5oQ^eHdDtQ7n# zY;>KX6B64KM3e1Cnt5zmtzFfN&D9x z-Y{0HVQ8Jih3sXVTA|I|PGzeR99#gHzaoXye5VUZ)xPFJ~G5l zV*Ob+o-D`axEkgitpg^)C>=euq7x^)5qRrAbYcPMSXRGg8(CqIAL|CQtsm*;kgj2q zdbD<&aBqBq^ceZXkU?&tC91R;***TYnYPLy^ed+FsbmiI#K5IR5%KaUFBr$dlFx zNgm0e86xfkj!2jO_05D`-RYPq!ZJOQ5gCW)lRY@17TKngIL7^{hR~QxNNMryqYwPp zv$`8hstEZd1z?Mzgcic^zZkDO{{IK#om>Ae#;eHp55yZxgv(efiPkb+*3j)O0s_5K zG2FgUM69&C7Lu`c`bVU*!DP;5lxpc*x4AKK!r2<5EeZ?%j|g;$^a1cbGn14Nr=~oa z^NnOInV|Shd>k#**nAn5m1P$mkK=$pOuhfJAmEn3gR4P1!E^GDMg~ywWqk}v-7-!#9^T<&}|KTV@ZIFhAaGCjL z7KKzV?7VkTcCm=hF(DRi%RwNggfrtTRH2j za>G*64NNm&S3wG6!_yG+b!vo>I#448UT0)Ejt521)nTn1; z=}o0BzU_)weRv5tPTvL;2kf$`Sznyw+1FY5aFxykW&MNy+@3q$*( z)-l$@TM$E}*OPQ-?>Mo?lG;L?p3a9-?4iof`sZ0>Vc>I|sjSd3xjPF)%3TXa&}(HKB@vDQC{4+s0F!M$SSd$nm9x8ehv{2DgPgb?`P{wVf^y1yK+B(F&rx zU7(I?K3~0hC&NIKy=~QYH{@Meagtj(SAnCR45Gr7E~esoz+~?)>@0N3{UrDkvLEn7 zO80d7!5YeK{@%00c&5>fHqf(h<72JsYT3W1+MU4|HB91ov)YakKMrGhn#nNVOY}DYg zXLPac5;>~$NKK^3;Dkvctyde7v>AvjA?`qodlPhkc1{|R&cnZIcRUTQ9UGWPX=9tU zO$r(#CJ{PlGXYARN*xh(zei9DnfF8sXWb)bf+i}9iF4hjhy8;xqFG98bunyJqNj^` zqY}!YiOK6}Ca#B4m#uO-!I^9R!?l+f3G6muh_(3K|LpZ-wcj60K0W6tV+IT<-d-wO z5#o>PmzdCFxvI=S(4UFC%YLJBd$HZ<&zoR|=bPxy52HUlYk&4YKS^uPg(rWWPe78T zw$ctqw8tcyyu;AAk~UYv6>Z+3J>pdmDPg)C-%s%Y^PuTafiuM_n-Gb z@K(fRSNBQFs#;d=ur9yQXIGvC6@}}`F}7A;?7VLAdA+z5@bDPP2ZfyxM(S3rlYWf! zw0=(>zf^xaPOREbGw$+!x!%h2dzbq2veoJC{m1vusKB4cuXz8j!H>KzfaUvlnLy8< zGX=nT=ZpXQng7=tLVsu6u?DQAndYB8-HUp73U}h$m)<;Z7yPbvLg*j!8D{7!S+*uY zJ`XI3N5lNvH9Ng5Q#HRPN7dtJ6%vmEo zbjBu`#3pe?#^@^DtanUl+r(Zh1^fY^9ES z+Yw%WPBDgS!Fq6CFzx`AOK{x$IaluDucJe`OgNM8bTh@AW z;eqBnsCZ;~VT3zD8d4sR#gn<|*YgG(G(%&gn|05`S*a-9*~fz9BS!<+df8vVnC;Wo zTdor2VcoBH1ZYM>UTR8yah#~2(7Fu_c{rv2o5_mL2F%@ZVjTQ$xLzI#gkxJE3}{@#tmv}AYRGBB&t234Hn&^%uyE#MNj(=QY*hkc9qzFsPws-GA04`vxE1;8&$q4w3ZESn!caH6n55g6aZ)8`nVNb29M{y z=8hkCm}a;UQjyt5&j4GCnJ8@hc+JQ-HO1jYq-vior~26C6w3L$P5OIYFiT+-1CO9x(7tBKXvK^*5fg`bym)R(Lo;%e1F z4aszvRVQ4P=uETO7B>FRNb!rIeM+5=ngTY}c9sl#FmCl2$K>vn;vf=tzSCH&{0l5r z>1`5ov(KRp2GQ_|92br-PhAhgyzC~$Vi zx57~QM$gqxF9s`y5d?u%SwY9(SEyW07N*au!rws^e9(DT-`G)PeXK1*zXeXlzwj9u zX+Bfk_;G$EVTSr4r!=4-u<$ciu$#GK(vDDHnJ72Zg>&b*fx~nXWbCjY?nK#Ko=jDm zGa=mcm`ytVsPyOCH_=wK&st;wHt`Xgr_)HI_sPUar~M>&E`R`kG;m37kL$F)y&{Ha!4Qp+aIjnG({(bRR@#p3T6~S_wWb zG|?T{)7k?nqw?AqlfeH2_TJc`7!4Fpw)^RjXA|h-e0B6js{`V>0PVPj)B4)zpq(3H zR>NTzE83iPHng78&>O*}z6xrr8fS#xZW5a9#DHi+4CPi^nnx`NFJXk`cq)mK*wtRh zK%O&`$m8i3#7H`_-h?OBi%xO2dLGk1&y!pK7;amcYxIm+=pe$IeLLn|l5;z~^Q%U` zG#a5m`=YP*U8UQ~!nJB$vcCmgXfBSZ6=hclhX~(=>>GCiq%F7?V|jZM(zTE_Fo8=e zY7VXVD9!=-ZLpiLz^M_s?QN^{)K7?Xj}0eGedlWng6z}2w`IAQtn_%!+j}NR27=bz zBV#stJ$|n@kAZtf%2CTnRjYc5B9Bru1&=9=_}<-6L&(>{)p32ycpt;fPXCEtnuoG+ zUlVJ952OEOHo9}ko1T@U7G=#LeIZ37eCq4gDQ$hwb$34j(baW-KLP!>g#n$F)+*k6a}=!}U+O{{gCZ#X8Gr9Mz~Mg%*b_CdcQ|`wIw~tY}Q4 z?%AhO<{(^Q4S8Uk`f{DT$-jVro&Mx!?{0y#hay2&qd;*C$vNxG*`aHV{mQ2ANjSvt_zm&U zj|%4Qy)d4Jt5hwTlQFGbv^R-TG!!4wXscawz%f73di~84n>F4Vs}JS0{L2t#a869ihdh_YMY%-AJN5d%goXcPFg0wAUZ`Og_&sCHmqzwAmMn*IJ5;2Vf*SeozR=`8 z5#5cWI7DYL`aWHp%b<9sR$th1Oomixeqd@dZ--1xr$|6W+$5na-&wT>b3&y9@FS)e zWnOm4YD1-e5@M+d}sXCM?ih7?#2&Q2L~LQKj8^OSNH`kARY__j$0=vNk3 zW-NYcSvwDp6YvrKZ0c_AIiUAc(Z1OINcGh2ip{|>J=N{eI}^@4^;D!?RnuDPRbnr% zig5MR$v)T^X^;B~=JUIZn!v!6)u7qZJ=D?D#bBy8cvo(q=)ozOKOV7lOB1r_n1_+w-T5sPsJ z(}l94^}wt!&mQ-SrUa`fNur#dr4kTpbfMNPE%7W8YXZtDP8LJ8gth4kcNm*u);eZD z-J=HCJ%v^&*WH;|8yfvx`GzC5;TpeA1eQ(CMIDAk;v$CeU|Olf7GVgHp7a{TZ}l1$ z?z+;auE}o5V7t<72FQb6l2ij;lAc0qUXnZ+G`Tz&G_w-B(l3kUYhRL@@gbW-KuACF zE|b~4U6s+uR2`PB!yXEt5~83?d$k6MIq4*#eo0Tdb#qEX8w+2d*%qsPsVv^%p{}Fb z%n>uV-wK71l0Q!aYR_<$?n+1^GZ!(5p^f29kP;@kmlMQU05DBYmVc+B}uEneC8itTN(RjFbA5()zwu$T-R=><`yLsQA_yb{hn3pe0~K z@_a6s)XN}+8(Lh>ImruS-m5y%y=Jc`4>^oRgp;lS%X5O|B*NtqPZyT&=Qc#_OqMgR z4ofG&0wXbsN%OF&_yN-?ZX` zS=~vjTo2-$DNbkoOx>bbuNQf6zQBt~Ve4$^*X$$2p!HCozZ556pf}m8JhN+dk(sS6 zfBPIs9t2)6a~ZfSkPa%Bp?+Hal+y2wSCH+%_*AhRC#3KYM*nF+rEs&M0b?U9FcV3? z_)8^{LgH#rB&f~8H}3aoZZyvXdis)8&O7SPN@bAXfE^hzIfDz&$UB)}jP-033p}w< zRW+`|%0Rw88|<5_x6@%u>+lL}U}_|07{9Cr?nD+Rm~`8B5#PipFyeud=@$Mg5#x3o zq-1t+?966%d3=rMZXD-dk3SkZ!>R?1=c*b68$!^ako8fUbFm%j3Nhi#iHzo(8*UsH z5>L9+n?W6`zkJZa*SuVk9V*bj-G8?=7_!lhY89Tjf^c25_>JKq`eb7lSkBXewFx|6 zM`zG=v8~*h>t@Y|=Mu`oF3;f5TbQdHQM8!vnKVa0NIEgC+UHSRjDdLqLcKdwh!<7- z8DA#yJ!_(L4M-jIKuJ|}RmDjqC?xPsk$&vJ#leji5->-ds6=MlRfUG-q*DbUsAM8S z5U6Xe_FQl;u{Dm!*BH%jS$LFLZp&?7@i3ar9ORM#n>rgrwe=x+;@z^kb=f!$)zjqD zO&kW}=CNR{S>()>*3pnXLF5l}8}IEtX(!XEz!AGFl81y{da~VoF%T8WpB|%!MvsfC zw?EJWnfbWhXd}rVK7;YWdD(l7vOkCe+8O!8?CSd&La{TV%L8tg;(H`qON{~G;)P>( zh0d-AbM|cI1k`OsgL3lZWi;u6H|o5W|-mcP_)1*-H|` z7y9eIJ)pYeL3FQb0K?Y`Mn}(U7T`L3&#QJ;qR(AxGIuxN@WRo-6pFSv!f$Yn*$Aii zbdj9l6kF_rK1|&0xwfVZbH36W^^yNiIk_@L5DmVQDDAZ~Z7gYk=#uw1cE9(y^yq^g zb7y*2IxPTP&{_i%b0r&X(4tduM)hThkv%4gRjma{^L<%n+)e&mi$U~rl#Q(V(u`Ox za3rx+6pRkf@Ix5LEVg zyu1I+B6h7d+q9BnnC*kUza#8SP=CN@t~b~0j^PpS?qf7vrB(gfjqeVqq8SVayQ^k~P zebS+@AegKY@(w9zYOOHfAtS8fl%ce!ET;{7o zkJjy;PszcX1Ak|_52ya_;2SyaB%r143x5C%DF`PIdp<{i^Yxg+!8iHy$9IRGl>@@* ztq85Hwu4RR_eT96ImEFaO~gjP=O7zxH9N0Sc+|f^z9)gNx$)xS)<%jJQ<{c?#j&sO z%Bwp-|A6N3Lv}5`>uc`8Wi+u$Vro=|kz!jaHU=4?C{+y``8Tr==>g0FLh#7lW&N2GZw9N@V&{W*UIzM)wmQCFWKIm~i_fE-LaN0H9 ziXiPstL1_RaJJT*E!XO@6gAY9>q_@%|@;#?%FdOgSR2nCoO@ zFS-8de*?k#_%J-$fVI~2onWMIf$Sumq<);#BR$=w&pwS4ZIoG(FwC zAU@u2#!=CO3!F$03HRXr&fLB)l+tXkI)|##cX0J?voY$8j{2fct8dY}O}JEMG}X`eGE|1LKf4QK zmnQP8xO;+yH-hNaX&L|uC=Z4w1ZmVyDr$$TIt5U67zcgsA#~_hI`A`fd-H6+Ad30y z-W_c+XhZ6sUR}Gg{lTfZE1h|nLrlHB=q_@`8hwyhNtnwcMZ>wqFn%GVL@e0KDuJ5? zvLMKw>F@m>obi&Rmk$u2y+LU9`A%#Pp!RZYpg9ksw~Fq%*gy4;j!o}ig%}7Cs^hno$=P7$9VP0e2B@A)*L5JtFB=sv7 zYqF|tDx=?dbEzJ8Xh=B`5rULbM%pI@PN;Lv4>Q%QVIPBrLuCx22MxVJZxBV&*|yJ1 zmv&|Na@#d+kjnu;LBM77c)+cj49Whc|~FC%`#i^w{&E zaP{)#)W2{{IJOh5w%c^P2K)R~J0+hu;_GET*c*^y0^BkU{_(AzvfrAxpFmoZC~zZ; zEq)PAN7#%mHm?@flGj0(95O~e)5POErBL7-+u6m-U8H4d0h6j0^B8n)0ywqS#Nr2g z8c5b$ymR5C3%lz6xDp@E=%R>BNQdMN0X8wqo&0K7Lmo)G$JMi zWqPz9Mi}T)x;Z!ftWhDqIFd%S1iYi8BZ0A~vArO>wKd3?lzv{$63vNHHWcL&Z-ABH zuUSl>>h@dM=!D5cEUb}Fl(+U%X~z(WmiG0H6nUxO>o8D1Z?Y>4NKLkMl8rgGX&{^3 zlw8YPjGD$jwo$j~AW?{3;z~tKCE%xnWO#GFZS$WgGT zTAwE-I|kHgzhNE6J0(4=qJVQbvz|Y_z}L>CK96|*kF~=js4*%5ZQvgX`UHUuqTXj-ojT=KNZ6T15+|+ ze+x|rYcZ$M!R3)e8}beJj3ZpjgiM$3dU>7>cKQYd#QLt;0i&*y^>iI=Ps<8gHo*L}L86IY(o}4PN zB`iSU4vb!siBLpM227;~hRgzYodw}?bamw|EtP_Df9Jxq72n$-yI?0dNd4M}jVBkc zLP~X4hUI6~@crY)Mps-qW_|tB7rfFi`V&4Nl}}*MIW9h}0JhsX?n|>9ki!_xnL-aM zAobN}W@}PRK^1PueldQ?=L^eL8osg?BI=f;*i))|88fIrmP@YgXaLsk!GIfuVAb0( z*rw5?DS)M^Qi{cpm*~J&Hpm=e2D5$2a z#9v-H*_b!z!UbKqz3Yy86_I+C^z%n3xkpV(eq$38xHl}n!_eJb#7?6(ap8k?r8RVI z=en;S;+CsCQ5<6Sf*@@7WH%|l)F}cYvMO4AfSQmG-N5TTIABQ~tpRN@UejjZ24<>c zp?f}M@WY+UjM>-^;*>oc87#~j4{;YcGWK;YOthmQUU^7!iZEt5ZJbcFD&(g%^;D%t zHVRQzyuxP2W02@C_`I_y`3f2qyXa)HD#0}_Zm%6<X?Ilu5=w@zGCddj8PX?|H#2!ZYRm|oCs97dI!T~jFX`rqUf zze5xPWX-Sjk`0iR6_N(9O+JOU6r{t|i4lE~Uxjz;*p*1$~0KpbXM%kKFoY z{3%A5qCGuc*tFj#HzjdB$=m$2I}x2q!Qv0*$;v-a88t0G82QQCpGXnN1HGjcTz)CY zd7*oyk*wlMFiS<8KZgR^5tDYLk$N8YKu;*{-0X~hGxbxD000aY_Gaun| z(`4q1z&ss|bPlAzfbEV5Kj~xIVP|TkMME!iG3D%?tAn~>i(tOF$Uo7`ikY>kEpCUN zO%0dqKhrnmQ-I>piBUw)-;K15KkW`ZA%~xDckT}F&%c63iw%QouCaUZhdFMZkW1nx z<8g!gD46=BO9=0#r4m8sGqLr<*I)j!-yy(ZsAk_ms+5dU2tbMk48u{dFMp*Qn zkqI@HX5#%lr{jw}HLjk~zr=d~BpO5%HT)_Zom}-EeCW*El_hyMA3;f6rZLGQaryCr zQlr~GrP3VENfl;^)@l*WX05HoV>fEl1!KLbBY1eqL6K#WBMs<7npFJl1F2e1;T0r6k+VOeI-IRM++P zh}*JKLFTi8svuh|b|(Q-z&OPPpcS#3=;GzrH<(D*xRTrKbOmDHg(>Fus}EN0=&)Pn z_TY4)T+jpTck4o_Y=kR4KIN?>+lJ0Nj~wFE??E>F>|k?tFTWE^Br+yUr?>2K^?wa*%L6oKjE*Ca0pqH|iq_ykl^(YATwA$`8g^u6GrA_CwFqSl~6h1AY zm$DVOk;Q$JwG^lOwMS*O9>W+qzmW8j79Ok(rhRw2v?N0=b5-)dCV5Fd+P`>G85u0fsHy&6Smv zkOfmmG&?>c(?{a2*P)wTv&qZlG)`vz>SMmF`k>;pDJduAQjDQ6>UJ8cp(x`RlPK?_ z{$|XXeVa;MMgdiAzmV&=$ubd!NqUautFSsSeRjG(v@_H^f^@?%@(~lRH)h%4bN8Y^ zmoRW4SvH*%3w69GX^>Osy02=xQASZ?rJx+RJ`B5jZ0%HKLYGVib&HkMTvzXDgv~Ec z6MF~LUY?68tN1Dt>zuF@5ZJlus+i4;#MQy3OhV?45%|N!e11&`A^GI9E zH>&z))g7zJfv*B_Vg8?bHA2(rWmAq|D8dHr)Uqt53Ch|FCY9LR3!@ONQbSkDi8l1A z0$TRL&jdQI-cIU0)ocCVIC)ztmw6u&jmD3Azym5mF7sSv<2`;4W5N*=2<0E?Z?fsT zk^I1*D6`>)Q=w1FNS0@DY86~bINwV`d>@N`Jj0u{KR12IaPwH1W##?ez4_yD_g6Yc zl!5xIkYn&jg#b%Y@=_Xfbnj+$`X|~``gi&XfQoJR&P}HW|6OeRp636FeB7}_WX_uz6~YWmK%??w-J5;m=&@Y|33ObNUL+*^|fk$(yRA-^EYs}lYI>4w&&;mMLL6e zYfG&?{LKu;*VBc*z1^&R?+&Y9vBnlh0nqm`^}oXU+vXdH8-L3DUNrF+1Lt~3*I*+N zLSt**Uvv4Qz&}fUe~E~wrqC^9MC`9({E!FA#O-b58M*l}6PN4{eN3A6YmNC3iyQ}s z#Y;&%izlTV2YJEbYLhUTTU1dv-?gtDt10e3TdUemMnH zIQJ4G5J1NgEnMi^#kXXO2!ll$N0ifc$Ju|l;+vvKie2A3VHb|$V_523)!DT5H9vSH z)wQ2NuNN(y{@K4A1Xac0wQ?FI4r_V2C4M+gj1vrjo$G;h_2*o3i*JF)=+kxNk*j)p zUR}@zloE@lx8k+zG-w9N@DB5mn4y2v4sM(I#e#~S>}p^1TUXV1IWFn;6o-@v+7Mi} zIf1-+l5S({p1fk^+n@liNojEvk7Q0n)ksDm{$VOw(Y2or&ausKnxe$&`&#ayW|NhD z`~{L#J3RVDREGXjtoBx(DK&~g$*Mgan-_Q`DftmrW9)C^(K`(Y_LxA%5j4w`gV*cY zEduKc1}&*!5_h95mjn?GSuzmgAU|YL_snA6hOse;{kdXkbjcXTKmVxoBEFe<;Nkc= zBj;QX1Fsr5B>5e>TVm7SruND*BS>pZU;2|x;AjTMKngdDcH#uDuCQgrCcmb%bb}X| z!z%4*{km?TAuIccM8J#QF=O6&nZ+)3v|=N}Dxh7?{C#9P&Qv^D#X#I?STp1pZCXX{ zy&?8-ri6g3EqHFhxUOiA_qJATpy{2(swiwO_xD9+Qg$6TaYh?OM0Q5v+i2YOUOr>< z7LdvWAkt|}qK6|d(>^5s#!QvA_{&!j-@G@D3}SgaF&P_R)$~9&WNb%&vHc=5sVS

%nSS&xDrA>nMr#AM@y0DiC`X95>Zw-g8B*h4O4EwyoMHt>09gtl&?jgObK94=$U z@^LId-bymF?LIr&RY`7@Y_siRG}jEv+e&nf5IFY-h3W-#>}o{b7wqCOTM<5 z*Ok65Tm*-;$O(=eR&$Os^Pgq%C^ZDlwao8hiA6EMhWFS8)ulz!{pNrq1TV=2QMyK# z60q}qy*zW>``)Q^2ctf#m5^KwWu&rIZI=uUxvSV~J!zt{b$a&)1TycU$GYqw@GS5| z5>Z`)AM0M?%bAgH#|GQv<6#e*K3<4Qx{B&Iy13X*TwsQX+kk5ZIqxS>E^ag2@frOr zwcuXhI}Sn-quWP&K`qL3UCK0yiVnEB+s!Sd{UdockM>}226yqkzR4WyTc@(obRArb z)5h~IczK!|PVC%@99-1uI+3a&mrH+@DH!FY(dyT>)Eb9xzkPFiXCGcXzxWw$|F#H$ zHSya00qM?qGoD=PLE-7nE4xS8@!I9U1g#4(_KXCgDV(A-X5cl~E!wmLV)Ql*dlpFTHJ+J7dFT0krug zzdG!|3)Z+=PgxOIx^e*(Bku59rkGBBgiTA?HWiL2sL5SvNa9gvNROX(W$Vz=b{626 z;qUaXKabws6)Voammi`K8U;d@#FM2%EotRrwB0zh1POpfV%HqR8QwaOVt_Jbh^x$& z7x;1qGC{AGaQ{YEooVA_87!<%+e5B9IRHx{Dz*j&p`gM8e=MVfHOUd;|{{`ZP^}SdP(o(GYYa@%VD_`}`dAKq^ zbH477wZF2w9o`)mnA&2*I;tS?IiFm~E}tBav8RGEn}j#=2D(|*S4rz$Stdig^2ZGAj9PaLK$CBa4^9Jj4mSjvnKbPt`$v;UyC70T z81Me*R7ylF4XPN0ok8W*I92NQa*_f#lC6bSS;N465{M!rZF#cLKdkl5h)Dn18;K`; z;0rR7$2zm-bHgOp%)-F^mY;Ja4tI_uekJija`s2=?qTFMe9>ATenAt?tD_6}9q9NL zGyM&!8@8{2F37&S(f>^*r;A0hsjx}^O(w53t(2#zxV{#EWvh~P{TuYTo1SyJ$SC6oEY z+y5h(3@i=FIoHRO)@C&Us^^+Dh#EBXy0)GlPP5xOPgL3-;L9b_L_;d)L-`BoRk0zb z-iK0??NyPPKzfBpIuLYx5&a19c`Be_WxSW{#xBD6LMjp5ur z$7`p=+lPJJ%$@&9CA(D*yu%_;_FFtAfS1sQjnaN(j-J8L6E&)hOwq@(LS$S!fDTW~ z(!M%kBZY-oJ*wVkh!*j1Da4z>X$4W31V-$7)}VIBg7lTB8*femq^)>ar(%G#wSo~j z8$P({>?rYTKS`@fPJe&JiL5HT^@s1!VW?f98i(x#k}a0i2nuOLbLhiF(vO!ikb)i? z8Hdg5uSKVOPA;?Agb#!fHWL|}V0Kj0dA3wsBvt|P25_cq5CB!}xC^|| zcA>gp<|tZ%e*&$w1xa~y1y)~8h4Rz<#vTOfM}}zBD%H6jy!wA(YSisNOdXAvi8{j_ z7@Z?mYFx!F4Uuszim)X;AOz0qxw`Y9ZvXiCn&3^Ly4=dQ9dJPZ(J~sl>*EKq^Nh(; zYVVEv+A_*s3uv@C`8q^zI6J&BHz_x;ZS`WGtZc3_^kQ>(86K=`NIck9u)DCe!-~|6 zZT0T0E*}ae{gyR`2LH-i$~gw25Db(K;kQacXbP&EL5+b}5l&L%v7H2fJ7gfG(FCOyA;T58KC@6o$y|A+N9`ng1x0-jC zjeBQy0a=6HSXPMKi}J|<%t2cnDjxRmd4&PsrTk-xM}A75JK-X>)bKEkjybDX-;9NA zRrOQSelH=H&RdnTL^rB37wjkck0?}N*S^3azVe~|c=bBc z{w`^p5dS^Uw0i@1MaDhli_~LXADwJPahdXSLbJtaQLF3*nU;nHzD^dIUpb9mwwB5l z&RS4K3411)v`y|TXYX#IRu-{#DRVP?2$rgdG=xZ>43PXTzj{V+mi3S{!gC82nnvp0 zrBWhK0=cG&e(lc2FVd+ZZAU}|R!$k>7zZi=#kJVQl+j^30FdcRT1QJF=TEtW0uX3MgFmGT+lTT80@(#(Nnd!<(G7q!$E$!#hoEZz^gjSb z2LPh*k|?mH$yG(^>=KwDI=>Fp&5AaS_$wnc0y_HGO3oSB4Z>G?{%TroIg=2>7LQOc ze<`Hc|1S!$ zM5pFq-zsaqNRo_Oj(`yP)P4N0$M~YNsnpU|jBdS{-qx%GeX(b2mWM@i!bH}P3+c9A z&X2=#0Mn!ED@X?tpNrkDtE;~xw;s1)M(_?0)onW8)&Qb92f;f%4)4$ycr#X`)Ch67 z4PmaFay`zLjXmcGlU-;o(+sSsi#0Ps>zpVpuX9UpzTY`4unk|~G8-Q#h-)Ytp|sm{ zpmzR|6HmJ()+J6PLEr<~`I*iO|C9K>5MvplgmRh9Ki-}Jo+a$%{BOTb#fD!MM?_a6 zZK6o8|Hf@Z*XyMu|HGGo|KQ7e!~caZl;KjG|HT);e$)Tvi_g&ihA-OQTca`H>usN- z2>%Ti+osi+ihsdH=ZJ8xtOxqxUvL2f0>FjC@E^F4PyGunAy@#o4B)yUT@3Seoa8d$RELGvu31S6TWLGEFm1RVTt(nHIy6ugp=$Ecv3f;JK z07^C#{&^e?pRq}|zt~3H|9E&-I`Is;UHmWpdU#BgjJ~PHE&x0{H0;%`@AUl*%m+Y| zr2lw$9<>ff>mWqYO7?iAf;LjNvG5r_C&2EctH*sMyAqhC|Lrui`@=XI0&p4{a$qju z&co&+n-Wv){^K+}W19F+r=cQuUm!$W{gA3M{%yquZYd(^$mj|LpAK+uEIiXR_~j&4 zp)Qf-zQ}kS040F$MP$69KRgK#%D^kZ02uNWhd^fSHyxfN$8RwFx1X7z4;%1+Wg^eQ z0+a7zOZyO5#&s78ex~dbbjOW6Ss(xS46V^t^gRTj#f!peDNUJp&3>)^%d^wWZ02TJ z%#@)Q{#j0DCQmXjrx@EKjh}3xh6S=%gcY2&3#~~=?2$_&W?uy4aJG#3((oTp5!?U2 zhsvzS5~f2uZk=K2uLurAk)P*0*89dfNFPS+6k|U)Z18G_cf4q|%S{IXYdCG%E!?c@ zA?m@tTZaqq!AOGjN$dM^R6cLzc8Ds{TXeI>S~G z?LcW&CXwVIcCx0#X&V03$f0C4{%YiHTmz-)hE&>clfOqz#f%kMiSx>xkKF7qC~oq1 z);)$F<{8pLetB7b%r!A#f$~*@kqh(zgX5G%yRr)^$OJ@xN)QDq;Z%`)&VR=w3${IY z{}>)%SL@KHwK0DF7#h%`7#Kc&kr;tbd{xLxyL4xgOTtv^Lyc_KvCwgv+(4d6tp_c# zeV(6?z^Z7gJlln1{@{uEvSk$QTQ|9^&Xoj7mtDY_bdte4OSf#NhF!rJKM zc0nEk^XOVO_T+s9XyhgO?lwL*u{?Eq2kz$IFM3wkb${$%^TC8 z%iUq!`24t|U!#^r^;|nQXXYL8^jRqJg9&2+b8>wIz@N%ZzYn~h_S#^nt}RmLm21_v>GyX)N^ZG?@e2AS%c_{v8{{Qfr|c`ViV z0}2|GwM{E<6CWLzt9q^>PH72K=HEdW z!G0>o^pIr03qwUg6KT$Lwz4LCvCaJ4Z&|ie)XRw6IBoI!_JK{SY#eVFz7ZD=K8R}< zJI|5v5{9__7ELq2I#0~{>}p>GrwGv_Cswu1=QPav>?Q5C;4zTQP-?%eSzOe=l% z;sryJk<6+Fy4LkPJz=i#*-thi9m zRgr&Dej1{@*nkA4kzU?s?ld*|(@AnM|0 zyO;U$ZbA{?ahB{1<7hJd1jl(HR&!J@CF~!N5TRvTt@Ld(c=%pyScMe_MYrh~%R}xD z5Z_@YhN%#L?!YftF69trn}hT;K#+296!8zAbrGXjC_-(_8Anzlw8@6*zAj@Dc$OBH8#~kqBt}vi%EfDk zMh7V5)0P%w3@vqoFqq+st@7G}=N3YUFmp0KGU|kAnnYgG?m>)RiNp~Fa(@D57)PnI zwU-pks!vi9Grr=iB2e4NF0(Hq2T89@>VP0XR1a!dROTm>52DyBbP*(D#t5nVSiC=0 z|0R(B#NC(d#7+^S374O9lL%(-#73RL8=@QGmj!+yxRDsbm0N{?b1kxcH+UrD;OeB1 zI%Cdh_o4`zPgF;}seut6S$_+Gt2y1pQHx7xvQ^eFgNgX>6mO%|Lldl?8;Jq>!|kYkao8qD=GR?xp=L3aHOe1ALm&$_p4v>@%NBX}k1Alg!i6dL7 z{`$hyV*ZVOxh;o-iJ;HHXF%(N7kZX)(ndt2R)(gWT!wA%lSAb-<@?nQaO)<)1; zS8uAx#Yu%NBx5cH*r^+~qb@Y4MO;Uw*OOu?W;u&F0+82m$>R^9ulXG`FLxuKwq-d9 z{yQz`kw)Ngf~SYFy7KONfYK{Xdw3fll7kU8nz49s-Et8<_iYjzZ{eS1R#_9xsj1GcB{|xHyYvwN#l7?-Nov+ONBL zN`VEO&0USyghVYnMiNmG;6MF8TJ?SDx}@8B%UXnrX6-#0Wy!BH0JFKSWZb(VKOoW**+}C z19Ew2Nas_=+7!NG3nVLgga$^3)GuI*6$Vu{%J|vG`+1Llkq-K|749$6=T7K%EGzLf(fyxS5NGL?PNq(SptShPZOrO$h@VMMKMfAI+@ zl(;WH27I6EtSoISYFk<2ixr8<2-t!rL=q>Q6kmNQD;|291|BlGMnfdcwBa=bh~ujx zYS=8erNt|C0^r6sK`BtgYVkrH(w@)t`^H9x_BoyxuUh$R)YD(4HzY z3O8bhNYh0vj2>jxWv_;#`3G*m7c+b3c%ta7G4> zknO);HSE%}yBwq7ceNB;qv(1@;!v>iGn;MtA8w=_EGrs9;XR}_e9Bu?d*Ea#4o2FE zCecfof2jT7(c_OV#cZ~GZwQsT4O62tY++1AIof12gI)X~ofT9KQOcaI7IWS7Xh>l* zt((2N=5&kBf!!tgg6_T(;T7EpMjFBPZI2hub7G0E7c?c6jgx@FQZz6}Jidc4AUq^9 z=|jFdm|JKgl%(T!fJtpF@Nm9XmZIpPHDwmxR;%gB3$nS6!NKGFB^3l+rQ(cKgwh~H zu6WB3vDk@vo@03Wp0!RW+ky$_L)=qq57I>pJz3}&9>0;IENCW*rJ~I?y6XhfA=`mJ zF+!E#_a>-xKT!YdVxlF~kg23m+xt1$I6Q4ckT)~dG#!$TPzF{|x{uhYiIW>|);Vs8 z0T4Tx>F+>O4^1^Y3H$8wR1U%H0^e5m;Dx6UlP;eW81hAU0_{+HX^^S=uhU<{E5uE& z+a%VBFW+ab1u4`CpO4GQ+%)p2SMs8vY}U|8S6~tLOMDjFXvO0ac&nfPM5GkK=+&LRR0|uDD{0 zSA8~qcf^-=;8!Q)|1N&iu=#B+!ZC+lJ2J$_q=5dufnRN|f?wmEsXRjZ1ycmA6AGXM z#7_;bpQ$EEx41Ml9ovNhtIiG@5ieI0%AVVAtFgbg=I273&xpV0ii!&56;%mNOwhDF z=V1t%sA5Ze)DkGGr0c7v4wPTDTElmNClWYgYYaN31Z+5JI{CS~4(V?8JaO=MQf(2n z&YD8*4}yl22b~s*z$f~?94Ce%rsA=-+TZ`6S3~v%fgs19N^?bD5w$eQ#bGW#CiYN~ z9(&v^bImP9tJ<;uQ_}h|g-Ljf;CYs20ePiUJ?{VZu`>%|pycoz@89(r9Bw(t3!!LU z5V7s|;o$dnG?=Bj^oDt}-R=ESmgl$d#}}}eeb0aA^X@b6>-MWm;Ln4=pR@QI|M$hb zKeyMPe}10ksZkLEUDkXndx!F1UsSIdN`ek|SInGE0q{7`EhJK&I zn3@4l^xw;0&$V-rU7($H0w-k6?6 zLRh@CoStg159v!GWctJ_3}F5Y^pE$z+#Hns4*utDCtaHCp22>{YQdKT3O1c^Tk-4J fzYEKojnCtDch6<_4=fPSpFbhT4Ddj^FhKtcZyF@) literal 0 HcmV?d00001 diff --git a/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml b/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml new file mode 100644 index 0000000000..11314116d5 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd +apiVersion: v2 +appVersion: 0.9.15 +description: Fleet Manager CustomResourceDefinitions +icon: https://charts.rancher.io/assets/logos/fleet.svg +name: fleet-crd +version: 103.1.13+up0.9.15 diff --git a/charts/fleet-crd/103.1.13+up0.9.15/README.md b/charts/fleet-crd/103.1.13+up0.9.15/README.md new file mode 100644 index 0000000000..2452ab2f1f --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/README.md @@ -0,0 +1,5 @@ +# Fleet CRD Helm Chart + +Fleet Manager CustomResourceDefinitions Helm chart is a requirement for the Fleet Helm Chart. + +The Fleet documentation is centralized in the [doc website](https://fleet.rancher.io/). \ No newline at end of file diff --git a/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml b/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml new file mode 100644 index 0000000000..d42811945d --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/templates/crds.yaml @@ -0,0 +1,6859 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundledeployments.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleDeployment + plural: bundledeployments + singular: bundledeployment + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.deployed + name: Deployed + type: string + - jsonPath: .status.display.monitored + name: Monitored + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'BundleDeployment is used internally by Fleet and should not + be used directly. + + When a Bundle is deployed to a cluster an instance of a Bundle is called + a + + BundleDeployment. A BundleDeployment represents the state of that Bundle + on + + a specific cluster with its cluster-specific customizations. The Fleet + agent + + is only aware of BundleDeployment resources that are created for the cluster + + the agent is managing.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + nullable: true + type: array + deploymentID: + description: DeploymentID is the ID of the currently applied deployment. + nullable: true + type: string + options: + description: Options are the deployment options, that are currently + applied. + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + paused: + description: 'Paused if set to true, will stop any BundleDeployments + from being + + updated. If true, BundleDeployments will be marked as out of sync + + when changes are detected.' + type: boolean + stagedDeploymentID: + description: StagedDeploymentID is the ID of the staged deployment. + nullable: true + type: string + stagedOptions: + description: 'StagedOptions are the deployment options, that are + staged for + + the next deployment.' + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state of + objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the + resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI + registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + nullable: true + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as + ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this + deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + type: object + status: + properties: + appliedDeploymentID: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + properties: + deployed: + nullable: true + type: string + monitored: + nullable: true + type: string + state: + nullable: true + type: string + type: object + modifiedStatus: + items: + description: 'ModifiedStatus is used to report the status of a + resource that is modified. + + It indicates if the modification was a create, a delete or a + patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + nonModified: + type: boolean + nonReadyStatus: + items: + description: NonReadyStatus is used to report the status of a + resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID values, including + UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names do not get + conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + ready: + type: boolean + release: + nullable: true + type: string + resources: + description: 'Resources lists the metadata of resources that were + deployed + + according to the helm release history.' + items: + description: BundleDeploymentResource contains the metadata of + a deployed resource. + properties: + apiVersion: + nullable: true + type: string + createdAt: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + nullable: true + type: array + syncGeneration: + nullable: true + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundlenamespacemappings.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: BundleNamespaceMapping + plural: bundlenamespacemappings + singular: bundlenamespacemapping + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BundleNamespaceMapping maps bundles to clusters in other namespaces. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + bundleSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + namespaceSelector: + description: 'A label selector is a label query over a set of resources. + The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches all objects. + A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship to + a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If the + operator is In or NotIn, + + the values array must be non-empty. If the operator is Exists + or DoesNotExist, + + the values array must be empty. This array is replaced during + a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose key + field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bundles.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Bundle + plural: bundles + singular: bundle + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Bundle contains the resources of an application and its deployment + options. + + It will be deployed as a Helm chart to target clusters. + + + + When a GitRepo is scanned it will produce one or more bundles. Bundles + are + + a collection of resources that get deployed to one or more cluster(s). + Bundle is the + + fundamental deployment unit used in Fleet. The contents of a Bundle may + be + + Kubernetes manifests, Kustomize configuration, or Helm charts. Regardless + + of the source the contents are dynamically rendered into a Helm chart + by + + the agent and installed into the downstream cluster as a Helm release.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for resources + that do not + + specify a namespace. This field is not used to enforce or lock + down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this will + also delete all your Custom Resources. + type: boolean + dependsOn: + description: DependsOn refers to the bundles which must be ready + before this bundle can be deployed. + items: + properties: + name: + description: Name of the bundle. + nullable: true + type: string + selector: + description: Selector matching bundle's labels. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + nullable: true + type: array + diff: + description: Diff can be used to ignore the modified state of objects + which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove fields + from the check for modifications. + items: + description: ComparePatch matches a resource and removes fields + from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the resource + to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain JSON + path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to match. + nullable: true + type: string + name: + description: Name is the name of the resource to match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from the resource. + items: + description: Operation of a ComparePatch, usually "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart name, + repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is performing + an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or OCI registry + based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's EnableDNS + option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. This + could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to deploy + the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + maxLength: 53 + nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + repo: + description: Repo is the name of the HTTPS helm repo to download + the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema validation + against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for its + own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to specify + the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps and + secrets. + items: + description: 'Define helm values that can come from configmap, + secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same namespace + as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo as ready. + It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when monitoring + the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be ignored + when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed resources + when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the dir + containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize resources. + This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will be appended + to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended to + the namespace created by Fleet. + nullable: true + type: object + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. It will be marked as out of sync. + type: boolean + resources: + description: 'Resources contains the resources that were read from + the bundle''s + + path. This includes the content of downloaded helm charts.' + items: + description: BundleResource represents the content of a single + resource from the bundle, like a YAML manifest. + properties: + content: + description: The content of the resource, can be compressed. + nullable: true + type: string + encoding: + description: Encoding is either empty or "base64+gz". + nullable: true + type: string + name: + description: Name of the resource, can include the bundle's + internal path. + nullable: true + type: string + type: object + nullable: true + type: array + rolloutStrategy: + description: 'RolloutStrategy controls the rollout of bundles, by + defining + + partitions, canaries and percentages for cluster availability.' + nullable: true + properties: + autoPartitionSize: + description: 'A number or percentage of how to automatically + partition clusters if no + + specific partitioning strategy is configured. + + default: 25%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailable: + description: 'A number or percentage of clusters that can be + unavailable during an update + + of a bundle. This follows the same basic approach as a deployment + rollout + + strategy. Once the number of clusters meets unavailable state + update will be + + paused. Default value is 100% which doesn''t take effect on + update. + + default: 100%' + nullable: true + x-kubernetes-int-or-string: true + maxUnavailablePartitions: + description: 'A number or percentage of cluster partitions that + can be unavailable during + + an update of a bundle. + + default: 0' + nullable: true + x-kubernetes-int-or-string: true + partitions: + description: 'A list of definitions of partitions. If any target + clusters do not match + + the configuration they are added to partitions at the end + following the + + autoPartitionSize.' + items: + description: Partition defines a separate rollout strategy + for a set of clusters. + properties: + clusterGroup: + description: A cluster group name to include in this partition + nullable: true + type: string + clusterGroupSelector: + description: Selector matching cluster group labels to + include in this partition + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: ClusterName is the name of a cluster to include + in this partition + nullable: true + type: string + clusterSelector: + description: Selector matching cluster labels to include + in this partition + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a + selector that contains values, a key, and an operator + that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and + DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the + operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + maxUnavailable: + description: 'A number or percentage of clusters that + can be unavailable in this + + partition before this partition is treated as done. + + default: 10%' + nullable: true + x-kubernetes-int-or-string: true + name: + description: A user-friendly name given to the partition + used for Display (optional). + nullable: true + type: string + type: object + nullable: true + type: array + type: object + serviceAccount: + description: ServiceAccount which will be used to perform this deployment. + nullable: true + type: string + targetRestrictions: + description: TargetRestrictions is an allow list, which controls + if a bundledeployment is created for a target. + items: + description: 'BundleTargetRestriction is used internally by Fleet + and should not be modified. + + It acts as an allow list, to prevent the creation of BundleDeployments + from + + Targets created by TargetCustomizations in fleet.yaml.' + properties: + clusterGroup: + nullable: true + type: string + clusterGroupSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + nullable: true + type: string + clusterSelector: + description: 'A label selector is a label query over a set + of resources. The result of matchLabels and + + matchExpressions are ANDed. An empty label selector matches + all objects. A null + + label selector matches no objects.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + name: + nullable: true + type: string + type: object + nullable: true + type: array + targets: + description: 'Targets refer to the clusters which will be deployed + to. + + Targets are evaluated in order and the first one to match is used.' + items: + description: 'BundleTarget declares clusters to deploy to. Fleet + will merge the + + BundleDeploymentOptions from customizations into this struct.' + properties: + clusterGroup: + description: ClusterGroup to match a specific cluster group + by name. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a selector to match cluster + groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: 'ClusterName to match a specific cluster by name + that will be + + selected' + nullable: true + type: string + clusterSelector: + description: 'ClusterSelector is a selector to match clusters. + The structure is + + the standard metav1.LabelSelector format. If clusterGroupSelector + or + + clusterGroup is specified, clusterSelector will be used + only to + + further refine the selection after clusterGroupSelector + and + + clusterGroup is evaluated.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will + be used if true. This will try to recreate all resources + in the release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + defaultNamespace: + description: 'DefaultNamespace is the namespace to use for + resources that do not + + specify a namespace. This field is not used to enforce or + lock down + + the deployment to a specific namespace.' + nullable: true + type: string + deleteCRDResources: + description: DeleteCRDResources deletes CRDs. Warning! this + will also delete all your Custom Resources. + type: boolean + diff: + description: Diff can be used to ignore the modified state + of objects which are amended at runtime. + nullable: true + properties: + comparePatches: + description: ComparePatches match a resource and remove + fields from the check for modifications. + items: + description: ComparePatch matches a resource and removes + fields from the check for modifications. + properties: + apiVersion: + description: APIVersion is the apiVersion of the + resource to match. + nullable: true + type: string + jsonPointers: + description: JSONPointers ignore diffs at a certain + JSON path. + items: + nullable: true + type: string + nullable: true + type: array + kind: + description: Kind is the kind of the resource to + match. + nullable: true + type: string + name: + description: Name is the name of the resource to + match. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource + to match. + nullable: true + type: string + operations: + description: Operations remove a JSON path from + the resource. + items: + description: Operation of a ComparePatch, usually + "remove". + properties: + op: + description: Op is usually "remove" + nullable: true + type: string + path: + description: Path is the JSON path to remove. + nullable: true + type: string + value: + description: Value is usually empty. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + doNotDeploy: + description: DoNotDeploy if set to true, will not deploy to + this target. + type: boolean + forceSyncGeneration: + description: ForceSyncGeneration is used to force a redeployment + type: integer + helm: + description: Helm options for the deployment, like the chart + name, repo and values. + nullable: true + properties: + atomic: + description: Atomic sets the --atomic flag when Helm is + performing an upgrade + type: boolean + chart: + description: 'Chart can refer to any go-getter URL or + OCI registry based helm + + chart URL. The chart will be downloaded.' + nullable: true + type: string + disableDNS: + description: DisableDNS can be used to customize Helm's + EnableDNS option, which Fleet sets to `true` by default. + type: boolean + disablePreProcess: + description: DisablePreProcess disables template processing + in values + type: boolean + force: + description: Force allows to override immutable resources. + This could be dangerous. + type: boolean + maxHistory: + description: MaxHistory limits the maximum number of revisions + saved per release by Helm. + type: integer + releaseName: + description: 'ReleaseName sets a custom release name to + deploy the chart as. If + + not specified a release name will be generated by combining + the + + invoking GitRepo.name + GitRepo.path.' + nullable: true + type: string + repo: + description: Repo is the name of the HTTPS helm repo to + download the chart from. + nullable: true + type: string + skipSchemaValidation: + description: SkipSchemaValidation allows skipping schema + validation against the chart values + type: boolean + takeOwnership: + description: TakeOwnership makes helm skip the check for + its own annotations + type: boolean + timeoutSeconds: + description: TimeoutSeconds is the time to wait for Helm + operations. + type: integer + values: + description: 'Values passed to Helm. It is possible to + specify the keys and values + + as go template strings.' + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + valuesFiles: + description: ValuesFiles is a list of files to load values + from. + items: + nullable: true + type: string + nullable: true + type: array + valuesFrom: + description: ValuesFrom loads the values from configmaps + and secrets. + items: + description: 'Define helm values that can come from + configmap, secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439' + properties: + configMapKeyRef: + description: The reference to a config map with + release values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + secretKeyRef: + description: The reference to a secret with release + values. + nullable: true + properties: + key: + nullable: true + type: string + name: + description: Name of a resource in the same + namespace as the referent. + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + version: + description: Version of the chart to download + nullable: true + type: string + waitForJobs: + description: 'WaitForJobs if set and timeoutSeconds provided, + will wait until all + + Jobs have been completed before marking the GitRepo + as ready. It + + will wait for as long as timeoutSeconds' + type: boolean + type: object + ignore: + description: IgnoreOptions can be used to ignore fields when + monitoring the bundle. + properties: + conditions: + description: Conditions is a list of conditions to be + ignored when monitoring the Bundle. + items: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + nullable: true + type: array + type: object + keepResources: + description: KeepResources can be used to keep the deployed + resources when removing the bundle + type: boolean + kustomize: + description: 'Kustomize options for the deployment, like the + dir containing the + + kustomization.yaml file.' + nullable: true + properties: + dir: + description: 'Dir points to a custom folder for kustomize + resources. This folder must contain + + a kustomization.yaml file.' + nullable: true + type: string + type: object + name: + description: 'Name of target. This value is largely for display + and logging. If + + not specified a default name of the format "target000" will + be used' + nullable: true + type: string + namespace: + description: 'TargetNamespace if present will assign all resource + to this + + namespace and if any cluster scoped resource exists the + deployment + + will fail.' + nullable: true + type: string + namespaceAnnotations: + additionalProperties: + nullable: true + type: string + description: NamespaceAnnotations are annotations that will + be appended to the namespace created by Fleet. + nullable: true + type: object + namespaceLabels: + additionalProperties: + nullable: true + type: string + description: NamespaceLabels are labels that will be appended + to the namespace created by Fleet. + nullable: true + type: object + serviceAccount: + description: ServiceAccount which will be used to perform + this deployment. + nullable: true + type: string + yaml: + description: 'YAML options, if using raw YAML these are names + that map to + + overlays/{name} files that will be used to replace or patch + a resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to + folders in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml + then a file + + ./overlays/myoverlay/subdir/resource.yaml will replace + the base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + nullable: true + type: array + yaml: + description: 'YAML options, if using raw YAML these are names that + map to + + overlays/{name} files that will be used to replace or patch a + resource.' + nullable: true + properties: + overlays: + description: 'Overlays is a list of names that maps to folders + in "overlays/". + + If you wish to customize the file ./subdir/resource.yaml then + a file + + ./overlays/myoverlay/subdir/resource.yaml will replace the + base + + file. + + A file named ./overlays/myoverlay/subdir/resource_patch.yaml + will patch the base file.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + status: + properties: + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the bundle.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: State is a summary state for the bundle, calculated + over the non-ready resources. + nullable: true + type: string + type: object + maxNew: + description: 'MaxNew is always 50. A bundle change can only stage + 50 + + bundledeployments at a time.' + type: integer + maxUnavailable: + description: 'MaxUnavailable is the maximum number of unavailable + deployments. See + + rollout configuration.' + type: integer + maxUnavailablePartitions: + description: 'MaxUnavailablePartitions is the maximum number of + unavailable + + partitions. The rollout configuration defines a maximum number + or + + percentage of unavailable partitions.' + type: integer + newlyCreated: + description: 'NewlyCreated is the number of bundle deployments that + have been created, + + not updated.' + type: integer + observedGeneration: + description: ObservedGeneration is the current generation of the + bundle. + type: integer + partitions: + description: PartitionStatus lists the status of each partition. + items: + description: PartitionStatus is the status of a single rollout + partition. + properties: + count: + description: Count is the number of clusters in the partition. + type: integer + maxUnavailable: + description: MaxUnavailable is the maximum number of unavailable + clusters in the partition. + type: integer + name: + description: Name is the name of the partition. + nullable: true + type: string + summary: + description: Summary is a summary state for the partition, + calculated over its non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, + but with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments + that have been deployed + + and for which all resources are ready, but where some + changes from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which + is filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information + about a bundle that is not ready for a + + given state like "ErrApplied". It contains a list + of non-ready or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, + like e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the + bundle is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for + each modified resource. + items: + description: 'ModifiedStatus is used to report + the status of a resource that is modified. + + It indicates if the modification was a create, + a delete or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for + each non-ready resource. + items: + description: NonReadyStatus is used to report + the status of a resource that is not ready. + It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique + ID values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias + to string. Being a type captures + + intent and helps make sure that UIDs and + names do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments + that have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream + agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments + that are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments + that have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, + but are waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: Unavailable is the number of unavailable clusters + in the partition. + type: integer + type: object + nullable: true + type: array + resourceKey: + description: 'ResourceKey lists resources, which will likely be + deployed. The + + actual list of resources on a cluster might differ, depending + on the + + helm chart, value templating, etc..' + items: + description: ResourceKey lists resources, which will likely be + deployed. + properties: + apiVersion: + description: APIVersion is the k8s api version of the resource. + nullable: true + type: string + kind: + description: Kind is the k8s api kind of the resource. + nullable: true + type: string + name: + description: Name is the name of the resource. + nullable: true + type: string + namespace: + description: Namespace is the namespace of the resource. + nullable: true + type: string + type: object + nullable: true + type: array + summary: + description: 'Summary contains the number of bundle deployments + in each state and + + a list of non-ready resources.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + unavailable: + description: 'Unavailable is the number of bundle deployments that + are not ready or + + where the AppliedDeploymentID in the status does not match the + + DeploymentID from the spec.' + type: integer + unavailablePartitions: + description: UnavailablePartitions is the number of unavailable + partitions. + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustergroups.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ClusterGroup + plural: clustergroups + singular: clustergroup + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyClusters + name: Clusters-Ready + type: string + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterGroup is a re-usable selector to target a group of clusters. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + selector: + description: Selector is a label selector, used to select clusters + for this group. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: 'A label selector requirement is a selector that + contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector applies + to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. If + the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is replaced + during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, whose + key field is "key", the + + operator is "In", and the values array contains only "value". + The requirements are ANDed.' + nullable: true + type: object + type: object + type: object + status: + properties: + clusterCount: + description: ClusterCount is the number of clusters in the cluster + group. + type: integer + conditions: + description: Conditions is a list of conditions and their statuses + for the cluster group. + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + display: + description: 'Display contains the number of ready, desiredready + clusters and a + + summary state for the bundle''s resources.' + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + nullable: true + type: string + readyClusters: + description: 'ReadyClusters is a string in the form "%d/%d", + that describes the + + number of clusters that are ready vs. the number of clusters + desired + + to be ready.' + nullable: true + type: string + state: + description: 'State is a summary state for the cluster group, + showing "NotReady" if + + there are non-ready resources.' + nullable: true + type: string + type: object + nonReadyClusterCount: + description: NonReadyClusterCount is the number of clusters that + are not ready. + type: integer + nonReadyClusters: + description: NonReadyClusters is a list of cluster names that are + not ready. + items: + nullable: true + type: string + nullable: true + type: array + resourceCounts: + description: 'ResourceCounts contains the number of resources in + each state over + + all bundles in the cluster group.' + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundle deployments and + their resources + + in the cluster group.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrations.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistration + plural: clusterregistrations + singular: clusterregistration + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.clusterName + name: Cluster-Name + type: string + - jsonPath: .spec.clusterLabels + name: Labels + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistration is used internally by Fleet and should + not be used directly. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + clientID: + description: 'ClientID is a unique string that will identify the + cluster. The + + agent either uses the configured ID or the kubeSystem.UID.' + nullable: true + type: string + clientRandom: + description: 'ClientRandom is a random string that the agent generates. + When + + fleet-controller grants a registration, it creates a registration + + secret with this string in the name.' + nullable: true + type: string + clusterLabels: + additionalProperties: + nullable: true + type: string + description: ClusterLabels are copied to the cluster resource during + the registration. + nullable: true + type: object + type: object + status: + properties: + clusterName: + description: 'ClusterName is only set after the registration is + being processed by + + fleet-controller.' + nullable: true + type: string + granted: + description: 'Granted is set to true, if the request service account + is present + + and its token secret exists. This happens directly before creating + + the registration secret, roles and rolebindings.' + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterregistrationtokens.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: ClusterRegistrationToken + plural: clusterregistrationtokens + singular: clusterregistrationtoken + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.secretName + name: Secret-Name + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterRegistrationToken is used by agents to register a new + cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + ttl: + description: 'TTL is the time to live for the token. It is used + to calculate the + + expiration time. If the token expires, it will be deleted.' + nullable: true + type: string + type: object + status: + properties: + expires: + description: Expires is the time when the token expires. + nullable: true + type: string + secretName: + description: SecretName is the name of the secret containing the + token. + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusters.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Cluster + plural: clusters + singular: cluster + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.display.readyBundles + name: Bundles-Ready + type: string + - jsonPath: .status.display.readyNodes + name: Nodes-Ready + type: string + - jsonPath: .status.display.sampleNode + name: Sample-Node + type: string + - jsonPath: .status.agent.lastSeen + name: Last-Seen + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Cluster corresponds to a Kubernetes cluster. Fleet deploys + bundles to targeted clusters. + + Clusters to which Fleet deploys manifests are referred to as downstream + + clusters. In the single cluster use case, the Fleet manager Kubernetes + + cluster is both the manager and downstream cluster at the same time.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + properties: + name: + maxLength: 63 + pattern: ^[-a-z0-9]+$ + type: string + type: object + spec: + properties: + agentAffinity: + description: 'AgentAffinity overrides the default affinity for the + cluster''s agent + + deployment. If this value is nil the default affinity is used.' + nullable: true + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the + pod. + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node matches the corresponding + matchExpressions; the + + node(s) with the highest sum are the most preferred.' + items: + description: 'An empty preferred scheduling term matches + all objects with implicit weight 0 + + (i.e. it''s a no-op). A null preferred scheduling term + matches no objects (i.e. is also a no-op).' + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to an update), + the system + + may or may not try to eventually evict the pod from its + node.' + nullable: true + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: 'A null or empty node selector term matches + no objects. The requirements of + + them are ANDed. + + The TopologySelectorTerm type implements a subset + of the NodeSelectorTerm.' + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: 'A node selector requirement is + a selector that contains values, a key, and + an operator + + that relates the key and values.' + properties: + key: + description: The label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'Represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + nullable: true + type: string + values: + description: 'An array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. If the + operator is Gt or Lt, the values + + array must have a single element, which + will be interpreted as an integer. + + This array is replaced during a strategic + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the affinity expressions specified by this field, but + it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the affinity requirements specified by + this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the affinity requirements specified by this field cease + to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods + to nodes that satisfy + + the anti-affinity expressions specified by this field, + but it may choose + + a node that violates one or more of the expressions. The + node that is + + most preferred is the one with the greatest sum of weights, + i.e. + + for each node that meets all of the scheduling requirements + (resource + + request, requiredDuringScheduling anti-affinity expressions, + etc.), + + compute a sum by iterating through the elements of this + field and adding + + "weight" to the sum if the node has pods which matches + the corresponding podAffinityTerm; the + + node(s) with the highest sum are the most preferred.' + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: 'A label selector requirement + is a selector that contains values, a + key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that + the selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a + key''s relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of + string values. If the operator is + In or NotIn, + + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + + the values array must be empty. This + array is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list + of namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, + where co-located is defined as running on a + node + + whose value of the label with key topologyKey + matches that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + weight: + description: 'weight associated with matching the + corresponding podAffinityTerm, + + in the range 1-100.' + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: 'If the anti-affinity requirements specified + by this field are not met at + + scheduling time, the pod will not be scheduled onto the + node. + + If the anti-affinity requirements specified by this field + cease to be met + + at some point during pod execution (e.g. due to a pod + label update), the + + system may or may not try to eventually evict the pod + from its node. + + When there are multiple elements, the lists of nodes corresponding + to each + + podAffinityTerm are intersected, i.e. all terms must be + satisfied.' + items: + description: 'Defines a set of pods (namely those matching + the labelSelector + + relative to the given namespace(s)) that this pod should + be + + co-located (affinity) or not co-located (anti-affinity) + with, + + where co-located is defined as running on a node whose + value of + + the label with key matches that of any + node on which + + a pod of the set of pods is running' + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaceSelector: + description: 'A label query over the set of namespaces + that the term applies to. + + The term is applied to the union of the namespaces + selected by this field + + and the ones listed in the namespaces field. + + null selector and null or empty namespaces list + means "this pod''s namespace". + + An empty selector ({}) matches all namespaces.' + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: 'A label selector requirement is + a selector that contains values, a key, and + an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the + selector applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s + relationship to a set of values. + + Valid operators are In, NotIn, Exists + and DoesNotExist.' + enum: + - In + - NotIn + - Exists + - DoesNotExist + nullable: true + type: string + values: + description: 'values is an array of string + values. If the operator is In or NotIn, + + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + + the values array must be empty. This array + is replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains + only "value". The requirements are ANDed.' + nullable: true + type: object + type: object + namespaces: + description: 'namespaces specifies a static list of + namespace names that the term applies to. + + The term is applied to the union of the namespaces + listed in this field + + and the ones selected by namespaceSelector. + + null or empty namespaces list and null namespaceSelector + means "this pod''s namespace".' + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + description: 'This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching + + the labelSelector in the specified namespaces, where + co-located is defined as running on a node + + whose value of the label with key topologyKey matches + that of any node on which any of the + + selected pods is running. + + Empty topologyKey is not allowed.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + agentEnvVars: + description: AgentEnvVars are extra environment variables to be + added to the agent deployment. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + nullable: true + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + + using the previously defined environment variables in the + container and + + any service environment variables. If a variable cannot + be resolved, + + the reference in the input string will be unchanged. Double + $$ are reduced + + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. + + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + + Escaped references will never be expanded, regardless of + whether the variable + + exists or not. + + Defaults to "".' + nullable: true + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + nullable: true + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + nullable: true + properties: + key: + description: The key to select. + nullable: true + type: string + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, + uid?' + nullable: true + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + nullable: true + type: boolean + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, + + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + nullable: true + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + nullable: true + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + nullable: true + type: string + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests + + (limits.cpu, limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + nullable: true + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + nullable: true + type: string + divisor: + description: Specifies the output format of the exposed + resources, defaults to "1" + nullable: true + type: string + resource: + description: 'Required: resource to select' + nullable: true + type: string + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + nullable: true + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + nullable: true + type: string + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, + uid?' + nullable: true + type: string + optional: + description: Specify whether the Secret or its key + must be defined + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + agentNamespace: + description: AgentNamespace defaults to the system namespace, e.g. + cattle-fleet-system. + nullable: true + type: string + agentResources: + description: AgentResources sets the resources for the cluster's + agent deployment. + nullable: true + properties: + claims: + description: 'Claims lists the names of resources, defined in + spec.resourceClaims, + + that are used by this container. + + + + This is an alpha field and requires enabling the + + DynamicResourceAllocation feature gate. + + + + This field is immutable. It can only be set for containers.' + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: 'Name must match the name of one entry in + pod.spec.resourceClaims of + + the Pod where this field is used. It makes that resource + available + + inside a container.' + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + description: 'Requests describes the minimum amount of compute + resources required. + + If Requests is omitted for a container, it defaults to Limits + if that is explicitly specified, + + otherwise to an implementation-defined value. Requests cannot + exceed Limits. + + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + nullable: true + type: object + type: object + agentTolerations: + description: AgentTolerations defines an extra set of Tolerations + to be added to the Agent deployment. + items: + description: 'The pod this Toleration is attached to tolerates + any taint that matches + + the triple using the matching operator .' + properties: + effect: + description: 'Effect indicates the taint effect to match. + Empty means match all taint effects. + + When specified, allowed values are NoSchedule, PreferNoSchedule + and NoExecute.' + nullable: true + type: string + key: + description: 'Key is the taint key that the toleration applies + to. Empty means match all taint keys. + + If the key is empty, operator must be Exists; this combination + means to match all values and all keys.' + nullable: true + type: string + operator: + description: 'Operator represents a key''s relationship to + the value. + + Valid operators are Exists and Equal. Defaults to Equal. + + Exists is equivalent to wildcard for value, so that a pod + can + + tolerate all taints of a particular category.' + nullable: true + type: string + tolerationSeconds: + description: 'TolerationSeconds represents the period of time + the toleration (which must be + + of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, + + it is not set, which means tolerate the taint forever (do + not evict). Zero and + + negative values will be treated as 0 (evict immediately) + by the system.' + maximum: 86400 + nullable: true + type: integer + value: + description: 'Value is the taint value the toleration matches + to. + + If the operator is Exists, the value should be empty, otherwise + just a regular string.' + nullable: true + type: string + type: object + nullable: true + type: array + clientID: + description: 'ClientID is a unique string that will identify the + cluster. It can + + either be predefined, or generated when importing the cluster.' + nullable: true + type: string + kubeConfigSecret: + description: 'KubeConfigSecret is the name of the secret containing + the kubeconfig for the downstream cluster. + + It can optionally contain a APIServerURL and CA to override the + + values in the fleet-controller''s configmap.' + nullable: true + type: string + kubeConfigSecretNamespace: + description: 'KubeConfigSecretNamespace is the namespace of the + secret containing the kubeconfig for the downstream cluster. + + If unset, it will be assumed the secret can be found in the namespace + that the Cluster object resides within.' + nullable: true + type: string + paused: + description: Paused if set to true, will stop any BundleDeployments + from being updated. + type: boolean + privateRepoURL: + description: PrivateRepoURL prefixes the image name and overrides + a global repo URL from the agents config. + nullable: true + type: string + redeployAgentGeneration: + description: RedeployAgentGeneration can be used to force redeploying + the agent. + type: integer + templateValues: + description: TemplateValues defines a cluster specific mapping of + values to be sent to fleet.yaml values templating. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + properties: + agent: + description: AgentStatus contains information about the agent. + properties: + lastSeen: + description: 'LastSeen is the last time the agent checked in + to update the status + + of the cluster resource.' + nullable: true + type: string + namespace: + description: Namespace is the namespace of the agent deployment, + e.g. "cattle-fleet-system". + nullable: true + type: string + nonReadyNodeNames: + description: 'NonReadyNode contains the names of non-ready nodes. + The list is + + limited to at most 3 names.' + items: + nullable: true + type: string + nullable: true + type: array + nonReadyNodes: + description: NonReadyNodes is the number of nodes that are not + ready. + type: integer + readyNodeNames: + description: 'ReadyNodes contains the names of ready nodes. + The list is limited to + + at most 3 names.' + items: + nullable: true + type: string + nullable: true + type: array + readyNodes: + description: ReadyNodes is the number of nodes that are ready. + type: integer + type: object + agentAffinityHash: + description: 'AgentAffinityHash is a hash of the agent''s affinity + configuration, + + used to detect changes.' + nullable: true + type: string + agentConfigChanged: + description: 'AgentConfigChanged is set to true if any of the agent + configuration + + changed, like the API server URL or CA. Setting it to true will + + trigger a re-import of the cluster.' + type: boolean + agentDeployedGeneration: + description: AgentDeployedGeneration is the generation of the agent + that is currently deployed. + nullable: true + type: integer + agentEnvVarsHash: + description: AgentEnvVarsHash is a hash of the agent's env vars, + used to detect changes. + nullable: true + type: string + agentMigrated: + description: 'AgentMigrated is always set to true after importing + a cluster. If + + false, it will trigger a migration. Old agents don''t have + + this in their status.' + type: boolean + agentNamespaceMigrated: + description: 'AgentNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents + + don''t have this in their status.' + type: boolean + agentPrivateRepoURL: + description: AgentPrivateRepoURL is the private repo URL for the + agent that is currently used. + nullable: true + type: string + agentResourcesHash: + description: 'AgentResourcesHash is a hash of the agent''s resources + configuration, + + used to detect changes.' + nullable: true + type: string + agentTLSMode: + description: 'AgentTLSMode supports two values: `system-store` and + `strict`. If set to + + `system-store`, instructs the agent to trust CA bundles from the + operating + + system''s store. If set to `strict`, then the agent shall only + connect to a + + server which uses the exact CA configured when creating/updating + the agent.' + nullable: true + type: string + agentTolerationsHash: + description: 'AgentTolerationsHash is a hash of the agent''s tolerations + + configuration, used to detect changes.' + nullable: true + type: string + apiServerCAHash: + description: APIServerCAHash is a hash of the upstream API server + CA, used to detect changes. + nullable: true + type: string + apiServerURL: + description: 'APIServerURL is the currently used URL of the API + server that the + + cluster uses to connect to upstream.' + nullable: true + type: string + cattleNamespaceMigrated: + description: 'CattleNamespaceMigrated is always set to true after + importing a + + cluster. If false, it will trigger a migration. Old Fleet agents, + + don''t have this in their status.' + type: boolean + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyGitRepos: + description: 'DesiredReadyGitRepos is the number of gitrepos for + this cluster that + + are desired to be ready.' + type: integer + display: + description: Display contains the number of ready bundles, nodes + and a summary state. + properties: + readyBundles: + description: 'ReadyBundles is a string in the form "%d/%d", + that describes the + + number of bundles that are ready vs. the number of bundles + desired + + to be ready.' + nullable: true + type: string + readyNodes: + description: 'ReadyNodes is a string in the form "%d/%d", that + describes the + + number of nodes that are ready vs. the number of expected + nodes.' + nullable: true + type: string + sampleNode: + description: 'SampleNode is the name of one of the nodes that + are ready. If no + + node is ready, it''s the name of a node that is not ready.' + nullable: true + type: string + state: + description: State of the cluster, either one of the bundle + states, or "WaitCheckIn". + nullable: true + type: string + type: object + namespace: + description: 'Namespace is the cluster namespace, it contains the + clusters service + + account as well as any bundledeployments. Example: + + "cluster-fleet-local-cluster-294db1acfa77-d9ccf852678f"' + nullable: true + type: string + readyGitRepos: + description: ReadyGitRepos is the number of gitrepos for this cluster + that are ready. + type: integer + resourceCounts: + description: ResourceCounts is an aggregate over the GitRepoResourceCounts. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + summary: + description: 'Summary is a summary of the bundledeployments. The + resource counts + + are copied from the gitrepo resource.' + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: contents.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: Content + plural: contents + singular: content + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Content is used internally by Fleet and should not be used + directly. It + + contains the resources from a bundle for a specific target cluster.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + content: + description: 'Content is a byte array, which contains the manifests + of a bundle. + + The bundle resources are copied into the bundledeployment''s content + + resource, so the downstream agent can deploy them.' + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitreporestrictions.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + kind: GitRepoRestriction + plural: gitreporestrictions + singular: gitreporestriction + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .defaultServiceAccount + name: Default-ServiceAccount + type: string + - jsonPath: .allowedServiceAccounts + name: Allowed-ServiceAccounts + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepoRestriction is a resource that can optionally be used + to restrict + + the options of GitRepos in the same namespace.' + properties: + allowedClientSecretNames: + description: AllowedClientSecretNames is a list of client secret names + that GitRepos are allowed to use. + items: + nullable: true + type: string + nullable: true + type: array + allowedRepoPatterns: + description: 'AllowedRepoPatterns is a list of regex patterns that restrict + the + + valid values of the Repo field of a GitRepo.' + items: + nullable: true + type: string + nullable: true + type: array + allowedServiceAccounts: + description: AllowedServiceAccounts is a list of service accounts that + GitRepos are allowed to use. + items: + nullable: true + type: string + nullable: true + type: array + allowedTargetNamespaces: + description: 'AllowedTargetNamespaces restricts TargetNamespace to the + given + + namespaces. If AllowedTargetNamespaces is set, TargetNamespace must + + be set.' + items: + nullable: true + type: string + nullable: true + type: array + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + defaultClientSecretName: + description: DefaultClientSecretName overrides the GitRepo's default + client secret. + nullable: true + type: string + defaultServiceAccount: + description: DefaultServiceAccount overrides the GitRepo's default service + account. + nullable: true + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepos.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: GitRepo + plural: gitrepos + singular: gitrepo + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.repo + name: Repo + type: string + - jsonPath: .status.commit + name: Commit + type: string + - jsonPath: .status.display.readyBundleDeployments + name: BundleDeployments-Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: 'GitRepo describes a git repository that is watched by Fleet. + + The resource contains the necessary information to deploy the repo, or + parts + + of it, to target clusters.' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + branch: + description: Branch The git branch to follow. + nullable: true + type: string + caBundle: + description: CABundle is a PEM encoded CA bundle which will be used + to validate the repo's certificate. + nullable: true + type: string + clientSecretName: + description: 'ClientSecretName is the name of the client secret + to be used to connect to the repo + + It is expected the secret be of type "kubernetes.io/basic-auth" + or "kubernetes.io/ssh-auth".' + nullable: true + type: string + correctDrift: + description: CorrectDrift specifies how drift correction should + work. + nullable: true + properties: + enabled: + description: Enabled correct drift if true. + type: boolean + force: + description: Force helm rollback with --force option will be + used if true. This will try to recreate all resources in the + release. + type: boolean + keepFailHistory: + description: KeepFailHistory keeps track of failed rollbacks + in the helm history. + type: boolean + type: object + forceSyncGeneration: + description: Increment this number to force a redeployment of contents + from Git. + type: integer + helmRepoURLRegex: + description: 'HelmRepoURLRegex Helm credentials will be used if + the helm repo matches this regex + + Credentials will always be used if this is empty or not provided.' + nullable: true + type: string + helmSecretName: + description: HelmSecretName contains the auth secret for a private + Helm repository. + nullable: true + type: string + helmSecretNameForPaths: + description: HelmSecretNameForPaths contains the auth secret for + private Helm repository for each path. + nullable: true + type: string + imageScanCommit: + description: Commit specifies how to commit to the git repo when + a new image is scanned and written back to git repo. + properties: + authorEmail: + description: AuthorEmail gives the email to provide when making + a commit + nullable: true + type: string + authorName: + description: AuthorName gives the name to provide when making + a commit + nullable: true + type: string + messageTemplate: + description: 'MessageTemplate provides a template for the commit + message, + + into which will be interpolated the details of the change + made.' + nullable: true + type: string + type: object + imageScanInterval: + description: ImageScanInterval is the interval of syncing scanned + images and writing back to git repo. + nullable: true + type: string + insecureSkipTLSVerify: + description: InsecureSkipTLSverify will use insecure HTTPS to clone + the repo. + type: boolean + keepResources: + description: KeepResources specifies if the resources created must + be kept after deleting the GitRepo. + type: boolean + paths: + description: 'Paths is the directories relative to the git repo + root that contain resources to be applied. + + Path globbing is supported, for example ["charts/*"] will match + all folders as a subdirectory of charts/ + + If empty, "/" is the default.' + items: + nullable: true + type: string + nullable: true + type: array + paused: + description: 'Paused, when true, causes changes in Git not to be + propagated down to the clusters but instead to mark + + resources as OutOfSync.' + type: boolean + pollingInterval: + description: PollingInterval is how often to check git for new updates. + nullable: true + type: string + repo: + description: Repo is a URL to a git repo to clone and index. + nullable: true + type: string + revision: + description: Revision A specific commit or tag to operate on. + nullable: true + type: string + serviceAccount: + description: ServiceAccount used in the downstream cluster for deployment. + nullable: true + type: string + targetNamespace: + description: 'Ensure that all resources are created in this namespace + + Any cluster scoped resource will be rejected if this is set + + Additionally this namespace will be created on demand.' + nullable: true + type: string + targets: + description: Targets is a list of targets this repo will deploy + to. + items: + description: GitTarget is a cluster or cluster group to deploy + to. + properties: + clusterGroup: + description: ClusterGroup is the name of a cluster group in + the same namespace as the clusters. + nullable: true + type: string + clusterGroupSelector: + description: ClusterGroupSelector is a label selector to select + cluster groups. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + clusterName: + description: ClusterName is the name of a cluster. + nullable: true + type: string + clusterSelector: + description: ClusterSelector is a label selector to select + clusters. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: 'A label selector requirement is a selector + that contains values, a key, and an operator that + + relates the key and values.' + properties: + key: + description: key is the label key that the selector + applies to. + nullable: true + type: string + operator: + description: 'operator represents a key''s relationship + to a set of values. + + Valid operators are In, NotIn, Exists and DoesNotExist.' + nullable: true + type: string + values: + description: 'values is an array of string values. + If the operator is In or NotIn, + + the values array must be non-empty. If the operator + is Exists or DoesNotExist, + + the values array must be empty. This array is + replaced during a strategic + + merge patch.' + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + description: 'matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels + + map is equivalent to an element of matchExpressions, + whose key field is "key", the + + operator is "In", and the values array contains only + "value". The requirements are ANDed.' + nullable: true + type: object + type: object + name: + description: Name is the name of this target. + nullable: true + type: string + type: object + nullable: true + type: array + type: object + status: + properties: + commit: + description: Commit is the Git commit hash from the last gitjob + run. + nullable: true + type: string + conditions: + description: 'Conditions is a list of Wrangler conditions that describe + the state + + of the GitRepo.' + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + desiredReadyClusters: + description: "DesiredReadyClusters\tis the number of clusters that\ + \ should be ready for bundles of this GitRepo." + type: integer + display: + description: Display contains a human readable summary of the status. + properties: + error: + description: Error is true if a message is present. + type: boolean + message: + description: Message contains the relevant message from the + deployment conditions. + nullable: true + type: string + readyBundleDeployments: + description: 'ReadyBundleDeployments is a string in the form + "%d/%d", that describes the + + number of ready bundledeployments over the total number of + bundledeployments.' + nullable: true + type: string + state: + description: 'State is the state of the GitRepo, e.g. "GitUpdating" + or the maximal + + BundleState according to StateRank.' + nullable: true + type: string + type: object + gitJobStatus: + description: GitJobStatus is the status of the last GitJob run, + e.g. "Current" if there was no error. + nullable: true + type: string + lastSyncedImageScanTime: + description: LastSyncedImageScanTime is the time of the last image + scan. + nullable: true + type: string + observedGeneration: + description: 'ObservedGeneration is the current generation of the + resource in the cluster. It is copied from k8s + + metadata.Generation. The value is incremented for all changes, + except for changes to .metadata or .status.' + type: integer + readyClusters: + description: 'ReadyClusters is the lowest number of clusters that + are ready over + + all the bundles of this GitRepo.' + type: integer + resourceCounts: + description: ResourceCounts contains the number of resources in + each state over all bundles. + properties: + desiredReady: + description: DesiredReady is the number of resources that should + be ready. + type: integer + missing: + description: Missing is the number of missing resources. + type: integer + modified: + description: Modified is the number of resources that have been + modified. + type: integer + notReady: + description: 'NotReady is the number of not ready resources. + Resources are not + + ready if they do not match any other state.' + type: integer + orphaned: + description: Orphaned is the number of orphaned resources. + type: integer + ready: + description: Ready is the number of ready resources. + type: integer + unknown: + description: Unknown is the number of resources in an unknown + state. + type: integer + waitApplied: + description: WaitApplied is the number of resources that are + waiting to be applied. + type: integer + type: object + resourceErrors: + description: ResourceErrors is a sorted list of errors from the + resources. + items: + nullable: true + type: string + nullable: true + type: array + resources: + description: Resources contains metadata about the resources of + each bundle. + items: + description: GitRepoResource contains metadata about the resources + of a bundle. + properties: + apiVersion: + description: APIVersion is the API version of the resource. + nullable: true + type: string + error: + description: Error is true if any Error in the PerClusterState + is true. + type: boolean + id: + description: ID is the name of the resource, e.g. "namespace1/my-config" + or "backingimagemanagers.storage.io". + nullable: true + type: string + incompleteState: + description: 'IncompleteState is true if a bundle summary + has 10 or more non-ready + + resources or a non-ready resource has more 10 or more non-ready + or + + modified states.' + type: boolean + kind: + description: Kind is the k8s kind of the resource. + nullable: true + type: string + message: + description: Message is the first message from the PerClusterStates. + nullable: true + type: string + name: + description: Name of the resource. + nullable: true + type: string + namespace: + description: Namespace of the resource. + nullable: true + type: string + perClusterState: + description: PerClusterState is a list of states for each + cluster. Derived from the summaries non-ready resources. + items: + description: ResourcePerClusterState is generated for each + non-ready resource of the bundles. + properties: + clusterId: + description: ClusterID is the id of the cluster. + nullable: true + type: string + error: + description: Error is true if the resource is in an + error state, copied from the bundle's summary for + non-ready resources. + type: boolean + message: + description: Message combines the messages from the + bundle's summary. Messages are joined with the delimiter + ';'. + nullable: true + type: string + patch: + description: Patch for modified resources. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + state: + description: State is the state of the resource. + nullable: true + type: string + transitioning: + description: 'Transitioning is true if the resource + is in a transitioning state, + + copied from the bundle''s summary for non-ready resources.' + type: boolean + type: object + nullable: true + type: array + state: + description: State is the state of the resource, e.g. "Unknown", + "WaitApplied", "ErrApplied" or "Ready". + nullable: true + type: string + transitioning: + description: Transitioning is true if any Transitioning in + the PerClusterState is true. + type: boolean + type: + description: Type is the type of the resource, e.g. "apiextensions.k8s.io.customresourcedefinition" + or "configmap". + nullable: true + type: string + type: object + nullable: true + type: array + summary: + description: Summary contains the number of bundle deployments in + each state and a list of non-ready resources. + properties: + desiredReady: + description: 'DesiredReady is the number of bundle deployments + that should be + + ready.' + type: integer + errApplied: + description: 'ErrApplied is the number of bundle deployments + that have been synced + + from the Fleet controller and the downstream cluster, but + with some + + errors when deploying the bundle.' + type: integer + modified: + description: 'Modified is the number of bundle deployments that + have been deployed + + and for which all resources are ready, but where some changes + from the + + Git repository have not yet been synced.' + type: integer + nonReadyResources: + description: 'NonReadyClusters is a list of states, which is + filled for a bundle + + that is not ready.' + items: + description: 'NonReadyResource contains information about + a bundle that is not ready for a + + given state like "ErrApplied". It contains a list of non-ready + or modified + + resources and their states.' + properties: + bundleState: + description: State is the state of the resource, like + e.g. "NotReady" or "ErrApplied". + nullable: true + type: string + message: + description: Message contains information why the bundle + is not ready. + nullable: true + type: string + modifiedStatus: + description: ModifiedStatus lists the state for each modified + resource. + items: + description: 'ModifiedStatus is used to report the status + of a resource that is modified. + + It indicates if the modification was a create, a delete + or a patch.' + properties: + apiVersion: + nullable: true + type: string + delete: + type: boolean + kind: + nullable: true + type: string + missing: + type: boolean + name: + nullable: true + type: string + namespace: + nullable: true + type: string + patch: + nullable: true + type: string + type: object + nullable: true + type: array + name: + description: Name is the name of the resource. + nullable: true + type: string + nonReadyStatus: + description: NonReadyStatus lists the state for each non-ready + resource. + items: + description: NonReadyStatus is used to report the status + of a resource that is not ready. It includes a summary. + properties: + apiVersion: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + summary: + properties: + error: + type: boolean + message: + items: + nullable: true + type: string + nullable: true + type: array + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + uid: + description: 'UID is a type that holds unique ID + values, including UUIDs. Because we + + don''t ONLY use UUIDs, this is an alias to string. Being + a type captures + + intent and helps make sure that UIDs and names + do not get conflated.' + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + notReady: + description: 'NotReady is the number of bundle deployments that + have been deployed + + where some resources are not ready.' + type: integer + outOfSync: + description: 'OutOfSync is the number of bundle deployments + that have been synced + + from Fleet controller, but not yet by the downstream agent.' + type: integer + pending: + description: 'Pending is the number of bundle deployments that + are being processed + + by Fleet controller.' + type: integer + ready: + description: 'Ready is the number of bundle deployments that + have been deployed + + where all resources are ready.' + type: integer + waitApplied: + description: 'WaitApplied is the number of bundle deployments + that have been + + synced from Fleet controller and downstream cluster, but are + waiting + + to be deployed.' + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: imagescans.fleet.cattle.io +spec: + group: fleet.cattle.io + names: + categories: + - fleet + kind: ImageScan + plural: imagescans + singular: imagescan + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.image + name: Repository + type: string + - jsonPath: .status.latestTag + name: Latest + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. + + Servers should convert recognized schemas to the latest internal value, + and + + may reject unrecognized values. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. + + Servers may infer this from the endpoint the client submits requests + to. + + Cannot be updated. + + In CamelCase. + + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: API is taken from https://github.com/fluxcd/image-reflector-controller + properties: + gitrepoName: + description: GitRepo reference name + nullable: true + type: string + image: + description: Image is the name of the image repository + nullable: true + type: string + interval: + description: 'Interval is the length of time to wait between + + scans of the image repository.' + nullable: true + type: string + policy: + description: 'Policy gives the particulars of the policy to be followed + in + + selecting the most recent image' + properties: + alphabetical: + description: Alphabetical set of rules to use for alphabetical + ordering of the tags. + nullable: true + properties: + order: + description: 'Order specifies the sorting order of the tags. + Given the letters of the + + alphabet as tags, ascending order would select Z, and + descending order + + would select A.' + nullable: true + type: string + type: object + semver: + description: 'SemVer gives a semantic version range to check + against the tags + + available.' + nullable: true + properties: + range: + description: 'Range gives a semver range for the image tag; + the highest + + version within the range that''s a tag yields the latest + image.' + nullable: true + type: string + type: object + type: object + secretRef: + description: 'SecretRef can be given the name of a secret containing + + credentials to use for the image registry. The secret should be + + created with `kubectl create secret docker-registry`, or the + + equivalent.' + nullable: true + properties: + name: + description: 'Name of the referent. + + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + + TODO: Add other useful fields. apiVersion, kind, uid?' + nullable: true + type: string + type: object + suspend: + description: 'This flag tells the controller to suspend subsequent + image scans. + + It does not apply to already started scans. Defaults to false.' + type: boolean + tagName: + description: TagName is the tag ref that needs to be put in manifest + to replace fields + nullable: true + type: string + type: object + status: + properties: + canonicalImageName: + description: 'CanonicalName is the name of the image repository + with all the + + implied bits made explicit; e.g., `docker.io/library/alpine` + + rather than `alpine`.' + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + nullable: true + type: string + lastUpdateTime: + description: The last time this condition was updated. + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + nullable: true + type: string + type: + description: Type of cluster condition. + nullable: true + type: string + type: object + nullable: true + type: array + lastScanTime: + description: LastScanTime is the last time image was scanned + nullable: true + type: string + latestDigest: + description: LatestDigest is the digest of latest tag + nullable: true + type: string + latestImage: + description: 'LatestImage gives the first in the list of images + scanned by + + the image repository, when filtered and ordered according to + + the policy.' + nullable: true + type: string + latestTag: + description: Latest tag is the latest tag filtered by the policy + nullable: true + type: string + observedGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml b/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml new file mode 100644 index 0000000000..b5296dbaf5 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/templates/gitjobs-crds.yaml @@ -0,0 +1,7690 @@ +{{- if .Capabilities.APIVersions.Has "apiextensions.k8s.io/v1" -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.git.repo + name: REPO + type: string + - jsonPath: .spec.git.branch + name: BRANCH + type: string + - jsonPath: .status.commit + name: COMMIT + type: string + - jsonPath: .status.jobStatus + name: JOBSTATUS + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + backoffLimitPerIndex: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + maxFailedIndexes: + nullable: true + type: integer + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + podReplacementPolicy: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- else -}} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: gitjobs.gitjob.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.git.repo + name: REPO + type: string + - JSONPath: .spec.git.branch + name: BRANCH + type: string + - JSONPath: .status.commit + name: COMMIT + type: string + - JSONPath: .status.jobStatus + name: JOBSTATUS + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: gitjob.cattle.io + names: + kind: GitJob + plural: gitjobs + singular: gitjob + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + forceUpdateGeneration: + type: integer + git: + properties: + branch: + nullable: true + type: string + caBundle: + nullable: true + type: string + clientSecretName: + nullable: true + type: string + insecureSkipTLSVerify: + type: boolean + onTag: + nullable: true + type: string + provider: + nullable: true + type: string + repo: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + jobSpec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + backoffLimit: + nullable: true + type: integer + backoffLimitPerIndex: + nullable: true + type: integer + completionMode: + nullable: true + type: string + completions: + nullable: true + type: integer + manualSelector: + nullable: true + type: boolean + maxFailedIndexes: + nullable: true + type: integer + parallelism: + nullable: true + type: integer + podFailurePolicy: + nullable: true + properties: + rules: + items: + properties: + action: + nullable: true + type: string + onExitCodes: + nullable: true + properties: + containerName: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + type: integer + nullable: true + type: array + type: object + onPodConditions: + items: + properties: + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + podReplacementPolicy: + nullable: true + type: string + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + suspend: + nullable: true + type: boolean + template: + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + activeDeadlineSeconds: + nullable: true + type: integer + affinity: + nullable: true + properties: + nodeAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + nullable: true + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchFields: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + type: object + nullable: true + type: array + type: object + type: object + podAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + podAntiAffinity: + nullable: true + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + weight: + type: integer + type: object + nullable: true + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaceSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + namespaces: + items: + nullable: true + type: string + nullable: true + type: array + topologyKey: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + type: object + automountServiceAccountToken: + nullable: true + type: boolean + containers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + dnsConfig: + nullable: true + properties: + nameservers: + items: + nullable: true + type: string + nullable: true + type: array + options: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + searches: + items: + nullable: true + type: string + nullable: true + type: array + type: object + dnsPolicy: + nullable: true + type: string + enableServiceLinks: + nullable: true + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + nullable: true + type: string + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + hostAliases: + items: + properties: + hostnames: + items: + nullable: true + type: string + nullable: true + type: array + ip: + nullable: true + type: string + type: object + nullable: true + type: array + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + nullable: true + type: boolean + hostname: + nullable: true + type: string + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + initContainers: + items: + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + env: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + imagePullPolicy: + nullable: true + type: string + lifecycle: + nullable: true + properties: + postStart: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + preStop: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + type: object + type: object + livenessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + name: + nullable: true + type: string + ports: + items: + properties: + containerPort: + type: integer + hostIP: + nullable: true + type: string + hostPort: + type: integer + name: + nullable: true + type: string + protocol: + nullable: true + type: string + type: object + nullable: true + type: array + readinessProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + nullable: true + type: string + restartPolicy: + nullable: true + type: string + type: object + nullable: true + type: array + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + restartPolicy: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + startupProbe: + nullable: true + properties: + exec: + nullable: true + properties: + command: + items: + nullable: true + type: string + nullable: true + type: array + type: object + failureThreshold: + type: integer + grpc: + nullable: true + properties: + port: + type: integer + service: + nullable: true + type: string + type: object + httpGet: + nullable: true + properties: + host: + nullable: true + type: string + httpHeaders: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + path: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + scheme: + nullable: true + type: string + type: object + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + tcpSocket: + nullable: true + properties: + host: + nullable: true + type: string + port: + x-kubernetes-int-or-string: true + type: object + terminationGracePeriodSeconds: + nullable: true + type: integer + timeoutSeconds: + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + nullable: true + type: string + terminationMessagePolicy: + nullable: true + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + nullable: true + type: string + name: + nullable: true + type: string + type: object + nullable: true + type: array + volumeMounts: + items: + properties: + mountPath: + nullable: true + type: string + mountPropagation: + nullable: true + type: string + name: + nullable: true + type: string + readOnly: + type: boolean + subPath: + nullable: true + type: string + subPathExpr: + nullable: true + type: string + type: object + nullable: true + type: array + workingDir: + nullable: true + type: string + type: object + nullable: true + type: array + nodeName: + nullable: true + type: string + nodeSelector: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + os: + nullable: true + properties: + name: + nullable: true + type: string + type: object + overhead: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + preemptionPolicy: + nullable: true + type: string + priority: + nullable: true + type: integer + priorityClassName: + nullable: true + type: string + readinessGates: + items: + properties: + conditionType: + nullable: true + type: string + type: object + nullable: true + type: array + resourceClaims: + items: + properties: + name: + nullable: true + type: string + source: + properties: + resourceClaimName: + nullable: true + type: string + resourceClaimTemplateName: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + restartPolicy: + nullable: true + type: string + runtimeClassName: + nullable: true + type: string + schedulerName: + nullable: true + type: string + schedulingGates: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + securityContext: + nullable: true + properties: + fsGroup: + nullable: true + type: integer + fsGroupChangePolicy: + nullable: true + type: string + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + supplementalGroups: + items: + type: integer + nullable: true + type: array + sysctls: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + type: object + nullable: true + type: array + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + serviceAccount: + nullable: true + type: string + serviceAccountName: + nullable: true + type: string + setHostnameAsFQDN: + nullable: true + type: boolean + shareProcessNamespace: + nullable: true + type: boolean + subdomain: + nullable: true + type: string + terminationGracePeriodSeconds: + nullable: true + type: integer + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + matchLabelKeys: + items: + nullable: true + type: string + nullable: true + type: array + maxSkew: + type: integer + minDomains: + nullable: true + type: integer + nodeAffinityPolicy: + nullable: true + type: string + nodeTaintsPolicy: + nullable: true + type: string + topologyKey: + nullable: true + type: string + whenUnsatisfiable: + nullable: true + type: string + type: object + nullable: true + type: array + volumes: + items: + properties: + awsElasticBlockStore: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + azureDisk: + nullable: true + properties: + cachingMode: + nullable: true + type: string + diskName: + nullable: true + type: string + diskURI: + nullable: true + type: string + fsType: + nullable: true + type: string + kind: + nullable: true + type: string + readOnly: + nullable: true + type: boolean + type: object + azureFile: + nullable: true + properties: + readOnly: + type: boolean + secretName: + nullable: true + type: string + shareName: + nullable: true + type: string + type: object + cephfs: + nullable: true + properties: + monitors: + items: + nullable: true + type: string + nullable: true + type: array + path: + nullable: true + type: string + readOnly: + type: boolean + secretFile: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + cinder: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeID: + nullable: true + type: string + type: object + configMap: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + csi: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + nodePublishSecretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + readOnly: + nullable: true + type: boolean + volumeAttributes: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + downwardAPI: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + emptyDir: + nullable: true + properties: + medium: + nullable: true + type: string + sizeLimit: + nullable: true + type: string + type: object + ephemeral: + nullable: true + properties: + volumeClaimTemplate: + nullable: true + properties: + metadata: + properties: + annotations: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + creationTimestamp: + nullable: true + type: string + deletionGracePeriodSeconds: + nullable: true + type: integer + deletionTimestamp: + nullable: true + type: string + finalizers: + items: + nullable: true + type: string + nullable: true + type: array + generateName: + nullable: true + type: string + generation: + type: integer + labels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + managedFields: + items: + properties: + apiVersion: + nullable: true + type: string + fieldsType: + nullable: true + type: string + fieldsV1: + nullable: true + type: object + manager: + nullable: true + type: string + operation: + nullable: true + type: string + subresource: + nullable: true + type: string + time: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + namespace: + nullable: true + type: string + ownerReferences: + items: + properties: + apiVersion: + nullable: true + type: string + blockOwnerDeletion: + nullable: true + type: boolean + controller: + nullable: true + type: boolean + kind: + nullable: true + type: string + name: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + nullable: true + type: array + resourceVersion: + nullable: true + type: string + selfLink: + nullable: true + type: string + uid: + nullable: true + type: string + type: object + spec: + properties: + accessModes: + items: + nullable: true + type: string + nullable: true + type: array + dataSource: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + type: object + dataSourceRef: + nullable: true + properties: + apiGroup: + nullable: true + type: string + kind: + nullable: true + type: string + name: + nullable: true + type: string + namespace: + nullable: true + type: string + type: object + resources: + properties: + claims: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + limits: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + requests: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + selector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + storageClassName: + nullable: true + type: string + volumeMode: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + type: object + type: object + fc: + nullable: true + properties: + fsType: + nullable: true + type: string + lun: + nullable: true + type: integer + readOnly: + type: boolean + targetWWNs: + items: + nullable: true + type: string + nullable: true + type: array + wwids: + items: + nullable: true + type: string + nullable: true + type: array + type: object + flexVolume: + nullable: true + properties: + driver: + nullable: true + type: string + fsType: + nullable: true + type: string + options: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + type: object + flocker: + nullable: true + properties: + datasetName: + nullable: true + type: string + datasetUUID: + nullable: true + type: string + type: object + gcePersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + partition: + type: integer + pdName: + nullable: true + type: string + readOnly: + type: boolean + type: object + gitRepo: + nullable: true + properties: + directory: + nullable: true + type: string + repository: + nullable: true + type: string + revision: + nullable: true + type: string + type: object + glusterfs: + nullable: true + properties: + endpoints: + nullable: true + type: string + path: + nullable: true + type: string + readOnly: + type: boolean + type: object + hostPath: + nullable: true + properties: + path: + nullable: true + type: string + type: + nullable: true + type: string + type: object + iscsi: + nullable: true + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + nullable: true + type: string + initiatorName: + nullable: true + type: string + iqn: + nullable: true + type: string + iscsiInterface: + nullable: true + type: string + lun: + type: integer + portals: + items: + nullable: true + type: string + nullable: true + type: array + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + targetPortal: + nullable: true + type: string + type: object + name: + nullable: true + type: string + nfs: + nullable: true + properties: + path: + nullable: true + type: string + readOnly: + type: boolean + server: + nullable: true + type: string + type: object + persistentVolumeClaim: + nullable: true + properties: + claimName: + nullable: true + type: string + readOnly: + type: boolean + type: object + photonPersistentDisk: + nullable: true + properties: + fsType: + nullable: true + type: string + pdID: + nullable: true + type: string + type: object + portworxVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + volumeID: + nullable: true + type: string + type: object + projected: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + sources: + items: + properties: + configMap: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + downwardAPI: + nullable: true + properties: + items: + items: + properties: + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + mode: + nullable: true + type: integer + path: + nullable: true + type: string + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + secret: + nullable: true + properties: + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + serviceAccountToken: + nullable: true + properties: + audience: + nullable: true + type: string + expirationSeconds: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + quobyte: + nullable: true + properties: + group: + nullable: true + type: string + readOnly: + type: boolean + registry: + nullable: true + type: string + tenant: + nullable: true + type: string + user: + nullable: true + type: string + volume: + nullable: true + type: string + type: object + rbd: + nullable: true + properties: + fsType: + nullable: true + type: string + image: + nullable: true + type: string + keyring: + nullable: true + type: string + monitors: + items: + nullable: true + type: string + nullable: true + type: array + pool: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + user: + nullable: true + type: string + type: object + scaleIO: + nullable: true + properties: + fsType: + nullable: true + type: string + gateway: + nullable: true + type: string + protectionDomain: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + sslEnabled: + type: boolean + storageMode: + nullable: true + type: string + storagePool: + nullable: true + type: string + system: + nullable: true + type: string + volumeName: + nullable: true + type: string + type: object + secret: + nullable: true + properties: + defaultMode: + nullable: true + type: integer + items: + items: + properties: + key: + nullable: true + type: string + mode: + nullable: true + type: integer + path: + nullable: true + type: string + type: object + nullable: true + type: array + optional: + nullable: true + type: boolean + secretName: + nullable: true + type: string + type: object + storageos: + nullable: true + properties: + fsType: + nullable: true + type: string + readOnly: + type: boolean + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + type: object + volumeName: + nullable: true + type: string + volumeNamespace: + nullable: true + type: string + type: object + vsphereVolume: + nullable: true + properties: + fsType: + nullable: true + type: string + storagePolicyID: + nullable: true + type: string + storagePolicyName: + nullable: true + type: string + volumePath: + nullable: true + type: string + type: object + type: object + nullable: true + type: array + type: object + type: object + ttlSecondsAfterFinished: + nullable: true + type: integer + type: object + syncInterval: + type: integer + type: object + status: + properties: + commit: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + event: + nullable: true + type: string + hookId: + nullable: true + type: string + jobStatus: + nullable: true + type: string + lastExecutedCommit: + nullable: true + type: string + lastSyncedTime: + nullable: true + type: string + observedGeneration: + type: integer + secretToken: + nullable: true + type: string + updateGeneration: + type: integer + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +{{- end -}} diff --git a/charts/fleet-crd/103.1.13+up0.9.15/values.yaml b/charts/fleet-crd/103.1.13+up0.9.15/values.yaml new file mode 100644 index 0000000000..d41d3a2444 --- /dev/null +++ b/charts/fleet-crd/103.1.13+up0.9.15/values.yaml @@ -0,0 +1 @@ +# This file is intentionally empty diff --git a/index.yaml b/index.yaml index 3382832577..d1f06356ed 100755 --- a/index.yaml +++ b/index.yaml @@ -2961,6 +2961,23 @@ entries: urls: - assets/fleet-crd/fleet-crd-104.0.0+up0.10.0.tgz version: 104.0.0+up0.10.0 + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-fleet-system + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/release-name: fleet-crd + apiVersion: v2 + appVersion: 0.9.15 + created: "2025-02-25T21:51:37.673595386-03:00" + description: Fleet Manager CustomResourceDefinitions + digest: f1db884b6121e13483da5b8e46ede45b763ce4594769caf3aca2c768eada2e39 + icon: https://charts.rancher.io/assets/logos/fleet.svg + name: fleet-crd + urls: + - assets/fleet-crd/fleet-crd-103.1.13+up0.9.15.tgz + version: 103.1.13+up0.9.15 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/hidden: "true" diff --git a/release.yaml b/release.yaml index e1e3dcb594..a07d4511dd 100644 --- a/release.yaml +++ b/release.yaml @@ -9,6 +9,7 @@ fleet: - 103.1.13+up0.9.15 fleet-crd: - 104.1.5+up0.10.9 + - 103.1.13+up0.9.15 fleet-agent: - 104.1.5+up0.10.9 - 103.1.13+up0.9.15 From 457432ba6329bdf3fb10efe29bd65a829d6b95a0 Mon Sep 17 00:00:00 2001 From: nicholasSUSE Date: Tue, 25 Feb 2025 21:52:40 -0300 Subject: [PATCH 16/19] fp: rancher-monitoring 103.2.2+up57.0.3 --- .../rancher-monitoring-103.2.2+up57.0.3.tgz | Bin 0 -> 480270 bytes .../103.2.2+up57.0.3/.editorconfig | 5 + .../103.2.2+up57.0.3/.helmignore | 29 + .../103.2.2+up57.0.3/CHANGELOG.md | 47 + .../103.2.2+up57.0.3/CONTRIBUTING.md | 12 + .../103.2.2+up57.0.3/Chart.yaml | 148 + .../103.2.2+up57.0.3/README.md | 1080 ++++ .../103.2.2+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../103.2.2+up57.0.3/charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../103.2.2+up57.0.3/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../103.2.2+up57.0.3/templates/NOTES.txt | 4 + .../103.2.2+up57.0.3/templates/_helpers.tpl | 467 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../ingress-nginx/network-policy.yaml | 19 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../103.2.2+up57.0.3/values.yaml | 5433 +++++++++++++++++ index.yaml | 152 + release.yaml | 1 + 573 files changed, 105021 insertions(+) create mode 100644 assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bc8ee547d89e63ecf27874f02411ab18c835a9de GIT binary patch literal 480270 zcmV)7K*zryiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPMcJ{OaJ#$>ia~gS~$l zPY%YD$-kiS2kL+0pIRy`{$>2$eYGd|8~KA7;|eRvnVjw*goUDWoT+6!ODP`}G$Txs zDLSlhwj{}3I_|5)F60zl?;wOwf*3Bz6af;UGhT~~KmkIvD5hwsR3)dQ(SkzrjF+QI z@RFz{sb!M!vaA_ZtI?7aWs+$iGJ^Z2l1dOAK8iSrIW|&Cn`umuq1XtyF0w5(N8szN+wxVPtn7J z@wk6Ae7%yCT&$+(;dnfLL3`_}RiziZsHGwzQJfdO)oQBGPsJHwDqOPu%Jv*JOLOp5 zI!^a@a7Dj&pDy=ya8)%w$LUv^0_3D3EGI0ZL{4`SlyR0*1yvRT&oRR(VR%-MdLB(|PW%{r?hQqvk!kH4hD2RB08D8{Feoe(KMzp5c zO-pl_mrqreuN#Rj+pcq4bBq2?1b@3)*C7;c*Q2YtMVDvUlH_%EdM8~8yyOf{8@2EjHR_W+u%myqp_{Dx-1Q1&%Zj>aM`$llGzptsfy zBU;yH)Ar~#z1Z&xw&-&Uyw}x^*+LL0UoI$n+uL1OVT*2C)o<#)|IGQ4EjrxhwCQ^5 zk6Ki^MUUGaGI2q+LaBGasCPi9b@;R$I&FtdHzCvQaOrlav=)=LM5Hb8Xd@ck0*kgq zBFoROrIW4l8#?aR4*K3tCp$TjnV?m3Nd-ck<}I=dlxe)nlth}j6O1S#QHdFyYu0-g zeQWqllw-M^aV&DVtEEAfYw6NLLZU8+L>VttCLFkngRMVX$2fp*&1Ynx! z6}0P*r6fv@3clcS)D?QtiY<07$m)uV9GHIxbteh#!JVYNAv^l__kkET@v>KmDN~p- zA^;#MF)gNgQ3bjB_n&H zRa#tNp_cSQ_O)_lO{-o+`*%sDqG_dk%Wq#`p(rE&NX~RoyVrmJ@j`G?U63>fgnevsE-K_$=sRwrZ-DRB&0~%XEqJe=I4v?3vDgD!N#SUk9B1Q+h1GL z_Idv4(f?lJq9*bKDu6fYe-HNe_YWSn^}l;x?%nBsKgFLRIwEsi7YdneM{_RvXCVm&of<3*w3g@zXLGu!g*nzc!(sdd#@Qpr4TngO`8(;EM@kSf(znqDF>yHD z>3*H=AY_@HDN>@=B)zFM1s(MeJy~M5AV|R)&h;{&%zYxrt5siJ5+OAIzy*A4e$P=4 zP27pp-_udgogHjtx=v-A%F7HuX>MzZ9*-Z7VbM?j@)Vuaa;aI)P^r9Oo$UZd;i~53 zGP&GKACD@WUEl@jV=errs+P-2@V9!g`viz+x#zf$dcCb0=>bSfo@g@!1%xUD#n zB&!91=^-rjVqQFw$SA(|b`T0wr-1&M?=>mC+>#ZPFlBetOx+qF+S{-HP-Z$kyP(zg zM9}#vDy}6tBjS=~f)^yc@a!fSAI#jU0y7dAIS^93AfmAbA(S98pOZ{Y(cmS2`j%w1vQ-fZ%p_0W zQmN#i+r)_u8`}h|p+k1)9)t_2gys*YXz*v)Dg((7D>u*1-tWbG?>55hK4q}=e|9^s z|3n+x+1zM@9m2UkeY@PQvw5pc?VV84+787Z_u#DzgtcE6UH9(Lb#H$wbe(LAuKS-C zx`G~JD^z{hhpLrTV??uQ@MnJ${`?wSzhjk4H_v!)|Esq>kHu)m0z%Z--T#!|#IJqFJph||H#6U6}luGiPvij|-D%b!7 zOwPCTl1QhkLtyij`lJ9OCw*MG!#F!7Ms@V&^tqg(zYRuq0Y(|lFRA2W@OO_7mqg5<(t;@eIYGFh-wIw= zP!v=b#A&6_-v$~%qyxSg=C8w4NVDb^Ax+1G_TSFWEv)kjtL6E5vn~pY1yNAZcLj6@etQD+A~_ipRCzZPHG_+3TwRhWF^Zl^=cc{(%CqVA4^|_xLa(_-7$0x*r9p{Al zxw9QNYa&1Os@ex*NH2O|kU5>2`<^+8T2dwS+ifHL1BGDCdZAg%}~ZLq>2<)F+b?sb_sf zP=*=GsUVpuRwxq$D`M8fsiBZs>WPEeLc#+0l>yg8{bfNf2}8K5z)Qh;j71&{?hf-Y zVd3fMAVKA5PfwX0mMN93!-A`-bxayx>75L($&YF+OW3|BTO2PK_n*xSC}(k7 zJy1};waH}o2SP2MzOD3bx0fjdbU{{AH0a`tCpUxMhS5qKvvrkXJn8Fy(9onV5XCEA z@WtvcFvxg-8Y?v7bz6CK?Cm%)fb21BTE8a|MOX%)Fgn7dqe|Pq~D{*)ByJtUS zp^p4fzL5|Fr*2zH5FHGNA1P>t zd3c!a-;swq^3Yw0JM!?mCl5PdnLG4OPrs^tZfDreh62hT&WaV9k!DNi95f?AHkGeo z&O>xOcgwJgsDd@q_yW6q7C9|7&}J(rb}DcS8F6qIH5-f3&d{$+GQ*PO$R$89+NYco z3Cb)>04$~$CAx_!;uT?FRmlXdh*Kk`JFO|do2Q*m2lFmYl6P))w|2Apq&A;Bce{U1 zce`P4Zl<*~u`lh*2PBLk_)ka{!>Qj)8y|o5_{*3ywzJi&A&_5gB9OnT;c7TUZ>ojB zIgvZH`G+3FaEMN8hU^!g(qTsCT(5Ie!UfW7%?xK3=#pZD(f{BxwB-DvClGWvM8_pw z5WOR{gug^>_$S-L4nkDdFbfgp#BjWTT#~{BcPPAoL~MKM#KdQIB4^>+w+)-G)6U4u z&<-0O(DBG%vWB~8zKTOrb;+Ev+%SK(=Z5D3h@l}U=DLB;eyPhGfwjS?)H{H(GE69NPUwmj1yW*V+%B-lmJl=C56E$}(TIYHW0oVz=ESu3BXlv-f|rOk zqgm63Ke|LC;x1ymxjoHr1M-u))ol37pMSA&(Do#n15Vij8Ft$--woTXae;2O(3(NN zZf|q2Vi2#rwuknfcdfN;`<;z2er){#47HAJzgjSe&3r>+=<7o7yUB+`bY^HDWr~PP zT%;W&39_p=axCLETF9uS*$F#K0us_zc#P$c0!y8dd(VNrfh|du=pA6BKOI{bb z-d+t5R@LoY1igVWXh8orLyq00h%$(ShkzxMbYrp%~`fj*}#D4R-kVHL5 z|9H)hbFoxZtkjsN!EQxt0^*T*~H^F~ZQuXuj0#|}gKdoyBEVS9c2sl6!w>wfyh zgFpDM5!azXS+lrn1MC5arsxlo2B+<++j7mJ1eoa({Pbi)II&rVFdRKizW@NZl=bF9 z(n$A%K0-ejm59_(UcP$$bgIurl#PH&L)muWb6HD;>MF;|WGd)M$QL9Ss70ZvPMEo1 ziUwVA)m+0qxq)GeJgM{u_0b2}*2b+%kT7Vi`o$yZwGkRkYsH&9QiztokC_V|k+*=m zMyPxZc+&Dgn40{%U%G&bHI)Wt6o*B@uR>P_$M~6wIbrB(Neg0pJ6!50OF3N^>wvcq zOP{=YCKXk+BK`)Q*F|AANh6%U#WZYP)Y^Rm*qpxTOyi=68)v*MIdkXq=U;Y)NM_{{ z%jFc|SvJ|*-x;EEWltH#7Z+s?z>iHP=KjNTdSrk@%BsFNnyi+Jmk0!BJudFhaxmv& z0Jr}@l1n0}S`GABvsVT*0MG!y2;O?w;F?nqp{#H6GfWGTpOS)Oa5B?Jf3B&@e%Yz} zf|N5NFHrd$zuUuSEwm2i|o0TH1zD{`l>WW1x z1yK$aHBp?0mzWlKMhn=+{yN{(f{Ac9TIJtSsrW+Ra$SY9g3DG}*JnR2FNkV=F|MJV zZ}zZ#+uhZuT#KBiB;z92vPfLZ+`xxb)gGiNIBgV6J*8)g3oSbJmI2{z6?y+{IC9<5 z`0{nb4HF@UZk$Gg-E!<~{d7`yPv2p2U)MoCp}vo=yqN0?Z34we+&d+Ct;HGZwP%GJ zJsKo4Ug?Ios$P1tfVqdJXpnv}uprKe0v_>vPBkSt*Mvp?r>QSM((LUZ~gG)0uYT%t48l$G<`LQs3}}C%vfg{Tss|LxHX#X zARgXgjItsbozturG+Kb}wbKEpWbL?n$TO^^BhL0mhcw&oo-o_rHTMi_7 zZK)y1J;e<%OfguYENd;@Vdlkk-CpLs)~79V(9!^En`;oia~IyCs&41_eDtw6C;(h%o`R1)hAm-#|QrA6?SfQ`qe=B%7U24bj}@ z$ZtHLa^nd-j9*!~JEnKX^zN9x^D%7>*x_O!$N~p5zlj=JYL`Tv4+b=NIVB z>2sN)H`1yPLO~CNS}LSM6&6Yhwfc)~sM{Gu9)(QMrIq=$^s2d$6htIb6_j_*tZzTc zv?p<6nv+Xk!SP2&RDx`}mcPht1utbn3W$^ABYwpilLG?GJ3Kx*B4DuyW!)oE36fz& za&(1PVZv%F1q-)32)H`4k5-B_dWObC#uN`l$yCdd!(fBrX_)O9Hm^CAt3<<1=CmOC zJF--CYJP+j&!NP9_adqPo!D_hUGvYT5Fzrm5@41>Uj!9Gea8oy{o1*|u;g-R#%`V! z>Y4PqDaRAuINi0A-B}cThKs3r5=a$JBQL9tuY$J@AY>F^Hc#v+Sx~9O$~&$Nhv=_H z(ug0Of|7S|aydv0_3Qe&Sb-)qJ$l8jLF|IBWE6|Q=b=W(HI&F$R(Wq$vEDqfw(oNX z!fBn+(@P>&eg(i33%bZ9z9a~1A*|68@;Mr(lXUz?Zwu69$e63{JNd?h1v~T6UV4x| zdf(`!ZFBQBnjEH^18Xpc&m}u=lgFKTPC(I5$V{ST-lE?J~`Z)P%eG7 zWU8>O`;48ohN)oeo#5zYgJr9_C~Pz#I-bAe>O>G}Y=MnCOrr%m{pRpVU}n?TLF{;A z0_18*gbZgHX~5u%u5!u)*P5=XO;LmGpaw>PNqW=Z1 zuVglfjO3+KJNaB`j6wTtK zE(%K&tc7#vS%!fHpkbMz51<@Qi>}sJVaL+**jFUDPa*ubb@c*mvKLO`x2SnX13YH7tK0Z?F8MNacIfvn{gKbCk<9&T2j1I(CoOGBpcmG>lJ8azrn^UaY|a~tocr#+Mi!S$kxA8lq4oyvZ0&G~f z&Y0~nNfzXlO|Min2w5^*S}Zo6emV8R$nsl&2EW zvaA&xV9hew70Tfd0>(H;R`AL?`o>Gy1<}>aFXQ!*n{Ijte2_*u_$QUJCUW%fVE^%> zfc6~u!G8mTUn8j?R8nSDlLc`>A|-zNhT-O`H*H0D7%zg!wLvd?S%pn!J_0iuMn8?x z1mMA+et1`8rZ_JGn1a{(??hjD3CZxF<2WlK7v z>RiMz2S(gF2bWJGT9Ie>oZ|xe8fAhin&IM}uVMVCZf^SjAnWvSP1&3`?a5DdGi=++ z+dA*X-(gX7;l10B-1dI^fum12Uf%$HjgHxzADivs;?oQ}xdHkO!_%nhBdtIgc0OB8 z(cglO{vMTHVXCHRe=MV~jnC#m>Cw3&SOq1My`?Y82)!g?ite$x@Q4Xx2{3$3HS6Zw znQA!G!C!TioG;9!;2?RoXme!*VEtoBuEsscogZ-o=ID;TNL5)C_ijUz(0weC7NXb! zGUo51N@#&db5dF$Gb&DKjFwm~QHd)fS=mXyT5D~VZjHJ$2d;?6&kH{9A;}Bxkz*9@I zx>pn??~{FTOYn+_Vugsza7A(?xy|adBrD*z7*|FaRSCFhXDb6tkGW@)w78VYrP-dG zT#|xUlr7G!z%+z6DigINiK>cZUNi7)y>WLo?zYC|bF(A21jM+~BB|lEM(@%1+&XUp z5Rc5U_k8c(*oQ6@N9LKd#)dN-qHk7CI@$FNoHJX`JBG>=MpT*d5X@6JI<-cBZO!C(jFzvy#u?oi*TMJu9ietSVRKO;@lWWqLN+R zht%w;X2QV}*TC!V%cw%Lq6Xs|m!#?9mK5USZ1r%cH#gcF8=qs;)aOq0g9Z1}qW<&q z&xw7w1Px??vy0TWNfmjk1}OREm!NKgFoUq#d07{VRs|WL6h#dZL|6_$dV5?0B<%Ey zzkPf9>dgsC&=FCX7V_`S+bgdE^%pL%D5Z&wq1MB};F%J1u^{?}z$je8G#(S3qlH5E z3&KzuxGJUJgVgr`jl-FJ0X3|P*z}9w`qZEVxP3=Z&bUw{KfzSU5Dr5B=YRiy=!>6! zM(Im^;r{Z=^b7QZ{t4ZpUw-+a1zWFY&l0yj(K$+=zEwmpT)a7b9u56DWfvNwff6{U zf-m&$u4@u>vePrqL*rDk6{cvyyB1I08Fd`gq-PoI1~g1fgj|QupPs%(3CzByW`Jxk zqC)nyZ`bPI4-4LNsUe&*=N@f$zgzc*TXf-GX%2g^Hk9cXD4c6g%?;oK6-+AYvc#e_ z3-iq)(+#z^?58AB;zhjbFU+@|etK$e*-!97chuKW@N##zzcsD4?9+NwoY$g=`aErX zK&D@yKes!Ln_W+3MFlR-8CT@I!uj;yZW;$Dxhx9wmz|PQ-+T8BKXX>ljG%!(7=tjv zU|?}*@Stb9o}#wtXW+ZGnTUR3!nxfivq+&va{E=ZM`3^l7Fh>q5TMZj4TAlB=S2-I zpTDJ*4QRfN0qWIw!M7oL#zknJfHW(UM-XIX!x*hHFiQ9mq@O1p`)K@VvMuN)UoT;y&12U-GYk=|^SW>k4qE}yfroq-2BtUSQ6?2;CX!G?45bfr zpDg;r3zM)~GJX(G%q;CT4gI2i$EA|qWN2N~ASr`QIjtvWF*tHh0rwXl{ApubL*qlv zOH3I&bMHDk!D=a;4Q|ebUv6VBO28;wVYPHe0F*qPc=R^@vQbr_N@+-eE&{Ys~Y&25&1J~WhJ$o~v6cLFYL;^kQQ>baT`J`4A zai`DCWQvZp+@-sL<923o(J#>J=VvXwNXPPX9G>f;^_%hdQ({)*!e_hAUA%u2XBupk zDq6wA0L$QTgss*WJ5hz3kwZWKvxR$B!gR>jYaxuLW%_O`PWsn$9d#O_*{XSXpnY~t zV|{EGSD@Lu5>zR|tRn_41i|^rWG9eLH3{o=JU6*BVzozi=sw;QGmnj+bOIeji7S1^ z2+o%AByA$DfG7@!@wzzzNyMqG>vq9#Q&ABMu?cj3D z!S=(>R>6i+PpTb+pMUOdQL2GhckN$(Nq+tr{lNlwS(h`Aq|&wRj)0mwmZ$Ls4cxzh zX!eY%!A#I;{E)=m{@_62sv;~mnpF*pu-xA?rkl_i)()rv9%g-5GZYWbG>+0gY`&ZB zu9y#W#n=l2aM>!|U7R%BK15h87wNchtliOfy@ymG&w@KAK!!7ukAmcmn)e<%yGtp0 zf|*A@#TNt=wjf98`2-JVg?FFnx=lgXq+{;?YHlEn2r{>Vw0?XNA6(7uce-*eEQ+H7g8LB|Nt#1!vg3>&?|bGaa6U9S z!c1lv@>U7#qpqM~w39(zoeSRo2Oj zaeaxq>k7Ee%)R}b`VIS(@j?CY=5!1XlPFzhFRTaZilcYM+~ zIM_kiu+u4;j3151!JTC(-l~elEV!1RING^|P~p-sdAHi;iV0Fq^~x44(td+{uvYtX zBUulxx-CfjyVCd$aeq(h9^AUOq4N)Yzt1_&XO?hSpCnE6K`kn8+4JCeK!twebm;>k zf6JBJqu`6!Uw2JHyyH1|-|Th)IU&Wdq)tu=fqO0>rV_>6brZx0eVXihgT5==CN}H6 zqMvug*)veaJf|gfN?NH8aAJp2Kmlin8J?5R?I3fXj4qP$FkA|)A%nrVmKmOZUD?$Ka;I z$W<$Z5PI5bbOm7xwG2jQl9GL*A!_;SIgH@l25YY7Pc>SwC9N6mN6b$tvnA17>UXbS zpV%Lvy21mBq*1u#eo%!pB<}}vlN;@*Cg9*rhNx7Oqw@9hGcyd=RnWV+xxZ7Y+MA-k zE{%w*FF+GBq6f8?_NgWM!-HIOe($$g!Dl8)=!j>v*CmW_Mbk1Lx_ctvDD1c*;^-$m z?a^!7g@Y(FpWf9fod9?Ws+=2FO5e6~d&~69IOBqF`7V=Gg)}2}>H;?xM^|SA3nCzV z;TbK+)X)1xn11xhh76y0u1JG7yJ%LcwOK#63Hf@TJ+%o%gDL+;lBsH0`NoyDtLm2e z*Iv`#e)iTbOY*djd2F~6UmQ)?NC<1h{wSY*6ThdqM16MVTF})DW6Lnv<5l_Lw>Iol zx=wdcXE05kW`$XfMUMFFN0O;~QXedwV}(=n!kqXy?;UXv=hMV=+Todm2AnBDXSL$O zIxLSQT_jj!OR7kwYC-g{lZ>-V!mO`TK6L#DWD9<~+;#c_4!u->BdY|j@B*twcMwU3 z70+dSk@DQZjW|-1DW1>Uaa;`t+$4(c&z?|HA-EuUXlTc6U_l=FuTuAXj$GpCX0;MS z<68h(1soqEh^#m>L3}|D8SIq5ouI$?qVxOD&0Xhua)AbwPvJfzh?=LK(0$`D3|C_4 znEL=WMYraRIF-5|M`MBs7qXTFY?_d|6RfI{ZP01%!OZ}J9Iv;?^%egA=IC&o}(s7pX%;%D}Vo998Pl{~P4`3DUMUD>lV2K(Bf~{63 zS{N4wV9%OgnZJ_R>*r@SLdXfi&giCZ?~*{m9xWbdUjFP8@hIVZuE@;ZWe8o7SwyhI zr3^T&nR_I{TnHr*3*rLTZ4hs?37T5Qhp7YiEPMyT_Wsn00T%V4hHNsiM(u+{+qN~3 zXcO^ojAmwV15@B4v)M{x>A*@HiL1j!`&+VSb;z3r@7b9CSNBZhhJ9-<7*kLfKIwAI zqJr=Zy(afpg5q)BP@GtE%KNQxw77y7WB_>|6z{C$ZPzT!oM2|5!B5+hbi;3Q`L^-0 zt;2SQkjji-60tH`J6i0Tr;)NH$u1DTVhu&Q0b^fw-htT)b(=>C*CMLzW*dC<&|;MJ zpEqG`cLKLY-)N*C28&xhYny5$dpoyT?;97|qp&fTy!*;^k+|F4)cbaY#5P9OATf`= z9eITc&FJ^?nrpDsR_et~y&Pt#5VisF>$i11Xg^J3&NYBy$Jf>aAIMoWnhQ}XffZR; z1IldGcXkaoAdLl7WtIV#DI$yM=2duMqL$BWCu>)umIS?GetZ3>)~-Pta3gVpZYqG) znl*aP7w8#;PTDqaBtZpV%*!TQ-UEH8f0Uf*f`wCI6ntSCPHp|bOtr8W(w919MwqaZDX~TN&ai^~XLBYL_b#II6&v5ZG>_1ySQ51Qu-A_Xd*Z zeQE~INiNa|V)4CqS~!H%>Q$PDPCd^{OM-I8A;z8#OY<&=W+f4bzH=@-dkD6{$=-N* z&t#|r!*k;gEyx0l>crDV*W=&D$>YEO@qQ_<<#lHdl8ssg1}(RLq8aQOT#fmZ<&SSmp942IeoriTN*Qp^9({Y%gu2z z@eYjWDPfak&Nyf*eb9u^RNQ0$?$)LAR)^BsE5)Velmsu4;M~Z==;-AcQ1aAeO$fXO zTGrt=qL@bWGzMtv+17GYSWpSF4j59FnB|3)e3}uPXpY@R?;l2&>eJ}VI#eP-t^@_C z9RQjH7G?%^-*&y;1ZSG;M9`>2QkggOYO?F*Fsqf1U$JIkB0V0M&4?mY!U@N=3r&H+ z)}ikeoAlF+K0fqKJjFC{XkBl0`bdVRt<5&p*(mT9{%sg;j7rF|c%Y91A;-E+S)(*; zaIw-OH;9{iro-iwSa49#nJg4U%(*B@zKh7)j8w`#Fudza zO|Fa=0nbgh(xx)c(K^e}W?w;Y{xHNzuUI%BT@=YLtuZwcpFafpP=QU@z8e66)Un3t z7GukJS=EpZ(sne8uf&KjBinJ~n`QFSX-YhPLH*aPsv$x2xJif;Qu46jq_<%ZI|b=p z6;0f${_vV^C4iT)&z#ys^M;UC1qnjO*- zi+sb7s6=15B!%xZpuRo_=1#PU;cfQaa64S$7sgu!Cb-SJ|9H-~ZK*h9iZ&jMv4tRp z*o5S1I${o~;uSCWV)YlY3f%&thQYk31pH}oz9GeTL!!Nl=9m_>Ac+m+>q$}j56;Mj zNxW``G(~Ys7thkjoKn&SbID~}YiIWd_wg#*9(A8cIL>{8pJsCPa(^w@8PXAN~7 zzek?qa>#a6L$I{LG|Cpo#-L(8o2Ql|IW;DGuZbvI3XVPo zF34)&rx89@t!28V1Que72Jmhm2Fo_kFb87N7 z9`To4xi#|@LHE_wS*_Jt*++>Z1FM%xZ4T`xoGJ2Fp(QVJskb088VHwT8)0Sa)HXA% zePEFV8NxNRgh`7{TOo5%%aUudzW%)o+prqvLBt}NaMWX zH#HENHFu+$cMWVVFw79H)siq}N8;QIHPg{?hG^SNZ70FG4?cQIus)>5ctxO}YP`}& z3RU(@o#@B(@~iY94g7b+B8t^clDd*g5L|w((SJG`P4>P@$LTnoOq&=gTUFOcY*f?N zW2iNrMqxd=mi5!=tGE|bOw-i=f6{zjZQrH%OAw7imJ(RvMFL>M*sA(2^ zLolkgJQXymy7pw`AP2)E28NaO_N@CA5hXR8(=N)XKuQquKu^P2*4=VB4?Ts=Lqv_> z55VG~L=!p@l*9D`A>qjIlZ}6p7Dpo%*;Wtqg}pxZC!~LQKykq5;W~w<90()|0)3u;Oeq)~j=PWN$pys8hQbB(jBM;eOPmodsus@g6`3`&wBZeFXWCXXkN(BCG1NB66GR?zH0 z6CIhtcUz~qg&~7!N{O1MTr5UQRTd*L&vd1TmiEt}P5y1QZ{Y?xb`cjrthbNCm=SeF z2y>SWR*IBWz$b{Fhqzo)&>)kXQaFe3#*L(mj9ps{r^g+nx5#d z4#52f@Z?8|9v+l(7wwPbE}E3PXtLZJmy-keAe?$UrwEB{4UIytO~hI_WgEn1qHe5V zMI&tB1ONHWHkeEARBO4c>Y~_ciz{3CH?Ya=c|~)7#zjWVlM(hnzu*wXC2K)i&QG2{ z%L&dDyt@*<=!Qz&612L`=4Qk5tWwYFYy3*Rv5(Y%CPdFhR@k&_b3v{LoCz*v=mTa; zWPewh&Jdhg%jCRwuI=Ogf-+jxWf)Opjcpp)jaC&SMy`Uy1QpMNun?0OMh{3I5$j_? zl?``oc(wY{0THTHnlOJf%_36vFe32Rey-$e)80(hOCqkQB)jI_nK0Q-r%`);+pokK7|4P=+E3pq zfvuNg7G@I9gk+{|#~pq9yn6j~>IP34q>&29gywV%0#Qb7uI*N= z!*m3Ci`0%1I4Drl7DfEpiC5kMJLRfVMh%|88-7lOpb_KTaMkBrEY6J=-uIMTSra>K zx~*)r^XCO$P!3 za(_n;DEYSfMz~^5+=4tu?j5BHFWew)v-2e{$+>TPZUa|TS&f>(`1dAlqeM4Kf0Wj3 z#Hov#F`v3eo)kx?oXzih0>gH&6~YQ(TgDGjN^FR^bD;4KrDgxk59t zqzulY9Z=0INp2>oIUKVw=Ba`Bqyuc<>#h%DS@~JJH>9AGWp5{72s7Ro-I)hZmT1v~ zU{+VJ*c%5U#&Io%{ev^&6}M}!-LPIBg|Q%0G!y(v>W?DK#HuXBT|`roqNlZH3P;cD zx1^l$S}fKc9ygRJs#WF~ubi4kxaI|3+%(4C6KMf7M-t~HWhYq5D|m>y61?=@D4k@( zVG0U&XSi%~Ei4Tc8oh}dhpwepR>6c?V4J9Kk9ix@$hP8DHs(w0c{{Jwd!)YVe=@ga zAhjK)ps#bi^;cDeOmGG~s9w=V_O&sJMpVankhHL8SUm|Izq97=X7OHZlYzIjbe$Zi zj*|*f@1iJGuUF<_wS?7l7f_f^&oU49c_Z3aTx?vCJ*tlDp#ID0$_s~K6)4)kDftTn`;{yU(7XqHk0n0{_M-9Q~s|A4QYc$Q5O}<9QEca41 zcyY$6uo`P?{FvYx5d}W0EWfdur9}aJVBFu(H}LF`jOCt&HQs;*{#lDViL}tuV7I zKHZ3#4cIe%2N1R7xeqA?%@JWa`l%+yZM?HJdMjQ)t0_8pb@n;{*p=#hPP!AG>pc%P zC9q8XsR03uY%PgIOM>%4->fbN<_{f5Uy}deHHQs|faMOld*zrhU+;rNGmPhiJl-dh zy*&FW+aq|gH+eLlJjP`2fMj3J9zEWF_~rQE@px2SECMK_Ad^YD;KS#82M5XXy+>c# z;h8x!!?$VSWRGyWhIF@fZ5>G-qGyn=8}#d727%mRlwFty5v=>N7r#IiX*ajgc%%Ui zSSyDnOOkR%a>xwcC`0uW&Vt}ugveXa)r!{wXXH3PBD$*%F@AJ?SEljYqI^HP<7IW zC8hUIA^m9Ffcpd~qz}f8;I?%JP2=OWgKm{4r=T-|#VQHqZOD!@N%tP6;{+EKWlqUs zxI%;(IyS`If91f|EZ>?ju$pAcZ29zLb;NPlEw9%}{MmZG>hA_szqcM_p&y1hkb!Tp?4Xuy7QaQ^86qhwwK}3r zMu#>ch7)ZrPt51rO)lX@z&!!*_c zaB!~*zJfdUfdg#wsyg*`yyX2#&5w|b*xWFo?-2@OMm?E{V=PS-4fb^-cgwqMjb1(a zYh-I|g-^!8lP&^sFNlDI_{TjLH`$Jw?2DOY2bLx#HrY|7j{Mo)#D+P#4pO zivke!B5)GpWyKjh0SGuH$ewP&dE-Cn3Qto}~J}uMV#FuJ^C^(}U}W*ALT!>o2drx_)&1_W&lCH0wz2DDULEwE=UD}j$ zE$qh)Htn;${{PfK`V*|FpH{>=o}b&e8;evO?lCY_Tga)r%%%???!U+LBethG=TkJp z;wDzCAV#HV|D7zk)K^7zm|s%KJ$qIdZ|1e`xAy&IqoUba&ukqkg|nqk!)^Ic=bvu5 zjnw&J%II3!OCKH>`xpFeq@19a?E|1%XuU6ptVpIF zxEtO3#>`fr6BX_FwIyd@f~}7#~#1$n>p5w&`awzr8SIzNf~C7xWEg!!OJ$FER)1rT;79qpi0n-czr*wFde-JL12bCsF8T{mc!@>U$>XO&!S7U{a*O}s2Ehs(a7?1bU}vA4yz$y|FIKG;2b_GeN$$Gm zfuCnpeSSZi%X769T&aTOUi4C2;o`dBI6udiMBoKEpN!?TwY0^xYDq-=>A}5i<~BmO zxWX%m{y_vsb6OAs{^Lz}F3;~%exB7rXbFt+bIp~srsKX|5xQ8a&YI}cCC?$Xer`WW zP%oUDlIYWwpjxUh-|UaD>86E>mnS^zB^RrYw@3#5`TY`C70J)_zFf;ITy3|hmEf5! z_-MnsuLpFV6PXEGDK6wSKM@>KAEy3?hDr8%A^MVKAbuxVSi1S#@4ZO}EV9bKmPc{vINariX2k_uzKZ`F-AD zhQ;aLU>ogkcj)xt(CN3Q-y04c9fJs5lQyA=p*5M?UD_*~*AmjuLBMy!IrZ>NNyn<{ zNL`;9*pDJ4)R9Imth647K4XjWZxl-mVY%Dj-|YrFG4F1Rx7_027vr$`x!OCH;f`ha zuq=bPBQv*tD){4=55q@>Tu_fS89wnPFPiY6Zp8i2C1Vil0JcaA<3>u zifv;yns5iu-`W>ryP+E0&?(a|s|$#JteH1Q8e;}Hv)aUaIm%ZImo$Uu^BI;VKZWT# zOao_)7$^_Z^0?mk+$*7B_>_fBC+Xf^C+p=D#>{bYXw|WKeW9z?3;CTF*zAcTq(M5=vFaLOj>301bI#hVqSrdQJ285iV;$rMr)6KZp~QK_kpUXLRN z5hS)+pC{oBJg_12k0%HF9Ujwv?Bg*n9?1kAQ30BqGkM2R-f@(79OWHH`73agU0QU< zg50qncPz+dL(aVBZzQbuI+bl>cK-4DqW3t5ZC#PRFRADk6A3U^QLLQ{~E-zL& zligMaYg!$=H|ui8sN69scZ|w!icz^^CGJ>>J67V(ha%h-%WrJJ7nkYB-cC1_195?1IfwjqVdE%b04FBP6W`?lL{j-G#^ea*>hJsKAszxU+ff?m3OoF=({A*Aw;?g?J{6O?0xcY!Gq zG&c@Q;!^J$%}GergC5pAg7C2znqTMo4d%O^tyyywf9T72o9p;}*qdK{h5MMCcUMrY zQBkc^Qr)DcTCb>DtE$pcL%%+xc^mKu&;YvfuYVYokw+E#&M?#g4oLswsjb@=*=;t1 zvc(ys{q~O`vifa5%b~yISX@!ImL(Yu(K9YGV#a%f<#NWc$k7w?;%e?c_!rs{xg-qH zISPl18nj1CtnrXApNrpVz~((2$#BuS%w2$ry1T?LWys>KtF(#Ht9TxXoWSnF4R9}l zanDdmy;qJtH_wH;H)MUXHK{Pm&1#h;X9&{P>FcaXzScZ6?^YmWs~fR78pS#W6y5|r z^bbfUC`mG27!f;d62zzQ%5h@Uj|q936?IO&{$Hc}Ik~*91kbMJN?u=LF)HY6l=JL@ zh*A1S|HZz#FiQXU;Qk-3M-Mi4HkXN7RV3)>KCTs?%j+3s^HQCcRA$$+YJP3fQST5&~v41YFmB1O9OQ0cZ-hYpD z9`7a#^e9R0q=Y*u;Z92UwWWmb_!TNRTcDaLTIlZ04D8E`=4~#ty#Vo<%MbsYjQtHy zQ*6Oy0%r<$tLE$FIh)J#`FA}u+jx<~^k{}9F^`uCqA+DX80qzrcxvW7izee; zG&b~lfyE3jh~o!45CM)LmsFGJ@2F{%Y7*!XGQ6^TE;v-ea(XhH$BpGK~Jg|wERUuzc_Pv2NMfM-BpPx-!3a&Sj z2#Kzi9Gq^=YM6w=FHcb==%vOK8_Z^nc8r)b&^XW0oC=IazhALqjRZxZ z*%;TXrwApQuQ)X#UwSNUkv^8SU=j^9iVk))*MSD6{Qz;bB;txnf(E*8GS-5I83i-RA;P=T`{fqvS1>N)M#c@si*i$S^I5?s&y(bVcPdehrv0ZkT|`_=m7q@GMXDUzOlL zrj(ELDooLdAZII<-BdBjNkJ4zDvPq;aei@DXBiM}0o`xH$=DS6XH6|?lM3(L&>G-Snsm#Zlsv;s**ZW=h(f+9L>H03+`p7K6Hf(?yLb$w+sF?5;y~=wBV||r!ksF8OD=sdyY?|oBb#mg` zGZCBto+ti_4!&DmU1A3^>YPYZerSrD&BwLkAjq9X6<+fT!osLh)b$K7f0zq2ZRXU~ zW(P~c+e;F@XWJccFw!2fi=0*?{$zLg_#_;dIYZ#)<+(u45W(3J1f#GTup)3N)49(E zp;i?f?|UzKPBfrTPXM4;o7zGd!P z<$-SZjcqsR>$nIi*sbm>Iy28oVYVRM!5tCC6_JL?pwge`oJjDT(^Q|=iXigVkO8Cw zp3iBfOTbbAStTGtph3FiO7j=q=ecj-4%TY}%CV?V41j2SZajKb!tV|RcU`OFlU)?9 zSYN*y$h2t??-G?tUwg+VdP4<~QcEk!@~JKTp5-I$mg(K{7i|dhru)8@L?nxv=ENi7 zXy_Ivq1DF4J3fhaa|n~Lx@sf)*>orVz_<&f%eJ&sq%vY#hUl=NJhzAKAh*d^1AeFQ252a@R5k@D1azWMVNr9yVM_L8%&q{Nao29MV6{k_cCv;<;Kfukm0#_p zWB*YuaYS9Kx+omcI-bAe>O>H!QU15ItHJrBmxK{1%^Q>LJuGbLi7vc}X|_s-*Goa< zk{9_D?FWn`RTLLTq`)h$I@oVkxgvt{yscD#^{*j+QvkIQW;(ivNNtXDx_)ND6@Q$m z9emW88HJj$tV#5*&Op^8$y(K0z#tB7v*ulWgtDrhqP_9B48EE7&EfM4>c6|5I;|)D zttZer5Q6PPb(*gO-iO&&%5v1)k&U2oU)I?YVTlg^dWN3< zpc#L#>-(CK4C^lJLo!KKU`BjzvlYP1;AR5HjhdNgQ0PGu^Lv^fcyfGnYUHK1(o=#e zNw>DlV|(?NT&kCbwd|@PER8T6hGuA_;WP9lf<===B1h{jYn!L5XrbT8dc9|m6~!6I9>hms&BC#k&5QYRqgyu_5HHa}pia=A)#CQ}0| zmGyj1-v(89rMl~ySS19@c_?r-t20G|y+Lmik~2B>8VBPZ^@KA;-Ud37x%^h}x|*Ud z4-WR58H<`7ZYX)l*(v9$b&W#N8%adFjU-vd%j!h%IW2EW$FOadDV_Oc(!wD4#Bvv_18g6Zq^{E^Q(l4!5!XqUHF z8@RO@<~}4+UCl)iGtEsBl49K1BnkM7HJDvC3Je zfv(7mSH_V6OlZcP%$bh+Y-G;wqH$EX4;X<8D%FE@ry*ZdqOw*sxUYb(y`+~`KFbMP zHRDLpnW2E*0RS^Z%1W&epZ!QO>po!PHJtxT#3}*Z@pkvn;%@^Jd~cEeRAUu!q5X*s z*ac+mu9%jh-(}VA>luejw1ZlU;4YWp5Iv)XA|jZM?y}HW-HJK!`b{hJ`K#V_fdiL6 zaMEEJX>}P_GV(>IO4H|BC~Aor2T!VXJvF|1R>n925yM*Ky<;`_URnObR!l2Ggq14X zDu<)CSoc?|mYN9!Eb`tqu3ZERdRyX4mx(Ndme|+`$`(oVeVj%=&FYhU`gr{Kp;wa{ zfDf7xed^()9&dW&IT))?;~sCs6b;;)-=s%LF*ET zCa?hT6PoOep1_Z8!#RvKR1LUW$uw_Lgb-lzr+${kJHtj47^^k&2}f+JFxN=qMY^r~ zLbFsK*JUA0ts1#1bX*TuC=P5VYi;eB&{)*Fsb2f!j+O^P$NS($zk^$O99WDG{0zq4 z&mi_O7>E9JT`uzoN_7(VKegck|-RK|C^$gh3 zC&1;}>hlEn&|Ux^%mbjcEP>^J3#)(ZO>bZoZ92Gx#k}j4-(%6gaWux>4$V8(`|ZXN zL+G*8-_lCI!9u^rj=QxDca8n_7Pi}7YrGB0JqyXQQQ~PzzuR(j(EN`T!L@*)*%9#K z#2#_OurUYRA>IcJA6k;4zuD>2!=o2Z(=zX6Xrdd>${C>`VaNH7(KB|st>Bs1t54sX zb^+a*S;i5TE?n+SPtk;1l(pc!2;i-=B)Q~-K@NgIjoO?%4dW~V+qz)dJ~MYI9Pd&% z{_3H{zt0qoXHf>B_f9&rCQ{gk@w$LZtHjwDDcVUd2!DO&7 z@~m^>>38Ajy6^P3?sVOD+Acd=yX)NGs?&1Q>37lTanI><%;_~2_BiD9JL9ae{PsHG z^qYLwnS0;S@#dqL$Kjl8cDC8Zw0fgiwa=s)np3?g)vNI@iBL2Hr6VMk=DOn`NO6tT zX$Se1CpA?sJGFD1fqBJ{;yaVDuAg6_iFaJ76dT(w383<4}z1D!1Z)L>0= z8DlWK@127jt15ynO|+-873*4YPY>M55N@Xsd5a;IpEDAIO1*i}f0B(2(VPxLnUkPO zQV{`30hoVP!fW%I^aCIhz$-axHYUT&F#st#3cN(Jsy6x)V0sKXn0f`tPa8)d&rhCN zvy`;=w;O~dbHVa9aHhcQXr2oM6E6JkLzaVZ;jBJEqwj6nflgJo43>?sA^qkkkUUrw zhXgJdro?X#O{IXI#{A;2#r<`AqHLR@UgCPtVn1&ip{8+k^`!3fZ+QJA?0wx!}=C6OJ zP3(gH=P5CIslhJvKhT6{;9RFn&o6jRrbrib^RRfGql*Osx*WLg-8#yrpEUMF>4c7Z zo1$}~0h^(@mzrWytZ}H}{QFdV!=Sb$6yIP;j$yh%S)lSGuuauvGL;Aqu&>X zv^JDzO(4=i7>9Jb7^Z_pNAVi`X^^BjNiyt^4(PDwI=zj_INuy^NyJQT`&UG&_8+E` zoz|X#6;mN>H227fU>fb`#!=}?i(Vw}q%ZP64;O(!uIo$G2=ce>F*>Ome4gIDhrq3= z-^pn$=w}ehxdF7vl_MK#jQ$C&0+?E|`R`wM1dt0NviV}Sue9Z`ktlvan60@aMVZRw z$U4QPmlHSU59C~Lc3@jakHFvIk{5N^xEes_;P1`1z{{Z10GafLJj+o<#|A}_fDeH`T9nG>vPK_tqF%mkh3%YYR83f(8^ zA~mX-UG!5;#A??dE@Z6~q|{I}_1m>L_n=9k1m01aarHg-sntGU62+WIx?mxXkU-zY z7;3g$|2Aw_s2dC|<*^UTX7jxB0P1*ydDc7x)Z?S}&*f=m-BZpwkLEcaH22#^o;atp zR(aA$H@;%0q~dwEpZ+(p5Im68;Vd`1vl-pxXYI$<<6O2S$@6nJcx-NX=cGhs#xIFj zAw>GJZUsSSGP~t8<`hHHO{qUcWvGL8AK_!EN?=qg2yJ_$10s zAAL3*?FAiu=Sv2tG2;iK+E@8AJI4`su;ZEEA!As7_z>F>Q z1%=OzC4ap~=x*TEh@4lW^X?1xA@l)tYpG}SGWxz35s2XF25V|RvnQ%a=|4m-3CU%Q z;8tDa5|d(yu1Ha|kHMzrLC`vnnA< z?FXo8?HhXOYpHoqDi!6ZeEs~)IUks=f^mBgQ}eKO1>tVq%E(n+JlDb+IGB~EeT-^H32o^-ZP4f&)B5pkI?QPNTX5qlUJKVNwL+57n z#1li}5rEjf5Sv`Xa9PX1RWQsS4ELG8jcNriONF5&?W6J3_WiCm1$s{(=HEJX{_Wk6 zk@NKyg#TS>`wf8&A3C0)nMSLl;HD_3RD>bU!Z{htdOUs{cqHlN{>^&BZ;4*F%5K>gZ4zu>UvkIWCwBZDMP|Gc-@I*f zPMGR*L);8PaRjO<@dCFIIXSM?(mpTkAS@$6VS+2_b)8;u z@4sA5Mnk*7XBDwmW<4vYT)yP$R1fjcQH!>g=-8$d$4q%SLe|1M>t>uW%u)D!kWyt<^Z?3Y%4I41Kdw_@kW5&@xp0V<5pMCr2p!%=D6b%B;bm;WzRR< zNx#?2C(?&kJV&|DsgGumRKHibzNO}Lb0Yttahh8}@Oe`4yy@F?9prn61#2o1Ez4Sg z?vk_2MvFVI5emSL)JwlYXsCB_0Pa)no^_4Lhn*-Q?rr3^J{b|H;{ zF}h-O%_yE^)ELKZg=!koYrQ!4bE1eSAxdq@ulx+5tH6P*ajLkcEAlt+OQGm{9% z5kYF>_?!_Vx&w+<$Pd6W2O>s>L=_D`Z*a%|+=Mt#$KEYZH7j9 z*AiX<0=qLa?1vqffpQ<@NEIn*J%lBeOOm4*mNY|J775)39yq(b8G{c)r9l)L1ortT zx&NQNcU^De#u7#Qx1Iur_H0WT@urI%Yft(+YTHRPZ6|6=?(|G2qZC;nSy8c2sRAXd zZSC*8#(BN-B7S?rP1bhw0IeHwV+3@j?-+s6MkDF2YczpSK@a^|6k1xqT{_7{2D?f770)G0%e(*eu3e74O=1z#60B!lh3QkJwJiP@A_{b4%Zl(&b3$w-Fxm$7x3?BOeS`!& zWf11UO0P-?t@dlzZuVCRDo9iAw;)7yI0+RY=FON{7X?C=$FsbSX0e7VvcNU2}-8i4~M&v zl`+gb*rELXg$~T(B=%*y?DvwX`jtiWmm#1l(D*Vk}m~NVKPm6Di=RCDG`;~19 zy`IQDwv@gsV9&iq-C5qoXcMnz6R+nF60c`tPPeK@!>6~c+e9+@tRfk?htn&c>QA3f z>a*2huHP*&#t7)_2=(P_LUPMmnk`aC5`D9I?qP?dT5lmyugt7o-TIhvTUE?oFyhhtao%R2BH%v4^3^}{^4 zl%c%}_LRz;1;vD4~_q<|aOn8GwC{a+*P2ea^dlet%3CcFfGv> zlmwBn3?*q2BqucDvHF*!LD%Hv(O6t@Kygm{;QkpUQ?|MKj~JnZ#IZ@25ejR*tnRym zbH;8HGWkE5SnK%)#5}BZX;fx@Xp;(Ax6aGf*A3oKRLr_|YdGP)$`WJ>!I)j0{_K3~@ zt@Nli34nGJF=Y{QlS(IS9kQwI4waxTB&fczA0t0%`>@;{nT!)~&+r!5s(COn5D z(vk5T_Iihdo{aiqN8xCZJM}Iyr%K6&Cu`~>R7{!*nks+JRQTjcnF{aAGp6$8t&+0r z%6Lyt*eXg%Or|#|PZ2oNgmKBpG4fa?IrnvP&sC*Al&1M z-ruGE-(C-J!RMNRau`z7-TkRyG9{)%cnuPbeSiwKxAF#=#1`^+aDOG_7agt~1Eda}-P{&QQd|3|fiJf+DuA|5Ao0YddG4{vM~)Xts7%`FT7K(!6hV zbF5%kJ%yMlXDJ?AH_qRABo#8Bv$2iYW@AT!IFTUYCCq4nZNF>I1Q5%m51Z&3ytFnW zyRAj>uE;2d$aSLUsOKd81lCHdSX%-977Pk?{1<)}EMlF@v5++xXdYMRhzW-hc>!va zm?h#%llcCtj$(-vAB%9&KLh&FaET|FJ3n`GrBP3vh$Jme$x8n9{tIB(> z9pxfcI0z09!t&FZk~^oS^_%c~5RC!csOhXAKGYv>)$(K{ z!GTtWRoz1K!Ll-;YTD>oZtz|PoaI5=Izl_y1p}-OTNU=J?w+|OOO|3uD0*?Xb&W8@ zb*VJ&d}-8q6R+y8UE$`t{8(Cpn2{Lk-D(e6{pb`oDuZ|A^k?O5YMZ}2*bQsv>+d#SCW#uzeL#oE90lQ`()p~V#8eDlnEziy>WZW^-XfDIgYOpKr z#VRyD)jrI~=;3m-fSrH$nRt6Po@L5<%E7G6wvHUTd=rO8_oC5KtHx^P6K*etM`35N zm&;ueM|kD^7xMFEo4EvWYQ&Bh_k-QZImEs9BHiF(dX3{!QlbVEo zoW6Q{^5)eK=f|&)Omjls1dqN=vuL!SSb3C?gp#|&mZvxXGpBqUKp@tTQ7-{W6IeQ+#idzEB)KV$>j&!cHK$i}mX;xS!hxF4iZcTAEie)i;G zY%SVXhnM&*Q|Yr+XeQbKB9LYz4W%3cL z=yOU!6eKk2s{FL$AVFz{Sq}v#$~D&+-;-$^jBQ1mLAFf=q{w@sG=`QrB4b7-w1eh2 zNQ{Jvx-H?_{8b9ECBNd4-l!^o=rvKc5p1|P0kc{MLX!|QT?AQ>Pz-URWKa+bHGvY_ zlocpNIr>Vb$6X2`Uz;6H2ZaSAPb6tghcwepi1UHR7LnEpdRr?=%5g#?9gNa8%&?3zAoyJ!+b@gl=A748Vrpw#N^k*^cyrs14j zQ-(p%Oklg@%PQ57sJ_1R%i45olY+S>1+%$5_#u4(7M_d|uZ-YwU#fnN6vIjB`}wQ4 zI0!9Ez(p2wM2`A>US!y3^(3O&@#d`p0CpWls0$66uOCkp%WlFLHh!c&uyF&VoPh%) zz1opW7x}Dg~ypCb*leT(Bbd0U=kz&3WIz$BEZwa$-(vwJN;}4lg&~J zBXqZyFBK^F0wfdvie!Z7f2ov)_WeU9KNMF)J9VK>Qn)`D=nRJO1TCSZc`DlsujPO2F6-`tX;*Jn+NC@r-9kzvGZTy&w zapxDaq%+%!T-uYTsi|fFW(br_AX(6KK^eOae$8iC#9oN>-AV?{*pdf9qQfF}XJn4mQdIcpyq~hPv1)ic zCAEB|b(vo0sH6@L;NCU-iTEQ}R-QNSVE=1RugzqJt-EgKKonD>L7!O?(Lgw!s%TTD z@2TnV$HOF8R6p;@AR+c^MN)bCXDh~%T~dd0l*w|9CZ%wwhyIg{+=?JNfbdQbr;B8B z)oN}?*o2&%OeWDdvU?Po8(;Y`uTT8ON9=;X?V)3548gup;VedJS7+b^KLMR6--<^c zL8>c=G|@ApGF%`BE(X$jY#!X`oH*c+!ugJYYHo1YLAC1WyA0~#6WHj0CRrX=EWU5=2^0JqYGvEf+u z>ejWmG{^{$fi`W;_zx;*Z-{aV4$T&+;#`k!(W@91JfSDtlO|CTB7WKDx$bs#kVrbi zH+rE$3X>^a=7%l@sWpIewy?@^^pz)*rBG#cclqAt)3f#CC`im7u!>C{Z)1$&>=hIWd4&yjt$~&s1gL9z^Ew~C=2Nk z1HcZ89Ak)t2udOFx%qWE#s+!gk&}cH1tZokNyJLY=||adlbm+I=Z2li8I}1gU~KI6 z3~Kn{L&j62vPRL-H>k(e@V)6cg#pfQ^;+VQU4P>au;Q;_$7JhSjU&|g@B!f6-dZ0; zGLqq7V78Bw&}^b~p`u=|w`B3^6uStsuyw95 zuscDK=rC+#V*BO(Mn=kj2I5C9W>cX7H?Riya@6Bwxhd^Pwxk@f?Xly%1mZ z*m^tq`p_ifes6G|B4LaZ${>XJ%HnaR7e?5%C%aEfXd(-uEAAIYHD;g+b7hyD4%QZ3 z?u^)FmpeOy8w00qx-&di*}~!*Z3^I8(S3qr#xwi%>%(*8c+M92gGa_K23K;3tB6Th zE*~=2Hml$S3e>WpGSl!EjNs@izem69^ zgcxgS#huXPX2jSU1&Pre5M_HuI2n4%LsIHVzt9;6QKJBq`fgM~E6bMnAB0%ivr<2A z&4{H$KnC!BK@y7JQ#2!S2=+5wZcL@Zi3e1e;cL3iAX$#;xHmG4UirKaWe_c`JR%{ zaoFI+4P}}c8H-!=*_Q`#{>hhcK@xef0(Mx?ybvow#72ZICt%{_D#}P=*^{0ddj%|I z-lsT=U`}Fg9U0un_z|pIkfk7@@eQmIvY>1iW|$ilGKvX_trIYw6?C)^z|d?17FTT> zqhNe_9b~Ywf$(x-U((JU>-%|PGdCsze{1b*PFha6m^_2&HnZOd`FVDPrsLFnuObiH zxpGw8jE~l9E|=qiMX{QMye02HK7IS+f4+KgesS_r35eOr&;OQpW62M69JmO?9Rem9 z{RYANdN;v5HlqFg2!@J6Vg5o%%ec9g3RxoK2}X%nj5^>RDFQJ2{Rl`pejnu!0Ye@e z(Znzc+w}qI3Cgalbp;(aU`wbTs%+3^Bes6xOJDD+g+=NyrH z6W7hL(mX9zu%I)XP?1tsq-+Kw@KehOSJ6uq-rAepd69;JvhRmProbKWvg2YBXpQR; zQl!db&!E%FvnR-=hRIGB*(2CR-Kp_|hn;8Fq5o|b85=+P0D%A7W2;SlB9Q{rU$@#dX1pl+oRUiX)r}n9*%@C+!`D; z#7@qA{#bSC{ogtNW|M#!JQo2b{iEtBO}QmeUoXrToTbW6-AFwhpmO~H69thWZ5X9j9`6x zb_`VVe3p^LbawV8H_!0v83xxDn6Ury%xr9eQ@LIDBO~q263$I z!c^WWv5mANt^{XhATZWL+3-5fj<5JchpHef9XbEGVjYaGbD0jAAd*GXDADQGxX=eK zZ9EALkFuYOo#OcFozEmF?hqn;!LNfG@YeDUx41D#xFgJPa{<3B#=wV9*xOY*eYLQ$RB>2k+~Qs)K&2ZO)F?4=ol82_$uX%f<|&82h$l!EqVEgWS2Q#h0&x0UvqEt z&>JNv9T7USBhm3tjN%Os#c3tS3!+S~A%0J1*Clpn{ECY`XLsFSVS!4!{IqtoY(sND zOk+gF^n&9PPK1Yf`4-z1`%COJGqRl6^Mcp;E~8xvSz{vvK{~YvDx7C8IH!PO&A8hf_*=k1C(p>nx&dBU*${==Y`3 zd?;=gj47a$v4N5Y>Y?KpJSu5)h2tBpE>LXnat1=ag1br*o|t>%G3sDe5Qn%g=w_m!KBC7AvL$c=bIA6URcv) zBzds1AX+__$J}2rPG4puO>xL`Bj$E;#6nU;$pnPF5<63L)-XHg-X%_{&R!uqu00+N z?lHN6P1n0Dpab0=-4rREK?QBLW|2S+ixPMOizr7^I3_bA*#(4?RrxOf04^P;9vTP- z(S&dZoSt>67HgP^0p zc7UxGvs09Sy(gd(#7gOB^ojSSFXGEDKq$!9a~`p4gxCR^x<(VL1t7K8d?KmIJ29py zVD}$!M%(jqR<<|v!ONN0 zt(}Tpb|~YN8d5b85`tHXa@di(jC}BOcN_?#qJ$(}*sV5rNLtG-nZWlMdP@#R?osaG z%%|6059em?RSWLj{m`lYmx&HAAXK|3GQ8N8YOvabF zN=bEvGa8LSaUkQ(;%@ieM2V9^Or|holP|LMu)#C3-9vtL9#@0sw+OZVSj5(js3}w&XF)!b?#^(M z$mV&(i98B1J0cYS-?QW}1GLSWG(uDqDu68);R)`9lN3X1hY!U(O5U=cXS{2gl|pYt zj0BY$i*cM_7!#UC)no30o1<-+Kk)dcGxX}OXFDPfKUW~W_~AsC7zc4QCOhK3hDMl2 z1%%)*B<`khkYLr;=msE;xgCtoiz9M%r6kOspp#Sd;^gI9?j{xr**ChzqZuKWxlC4p zu1R*OvP$w>$_%3!$?20EC781B#vMJH6r>-N@bf#TtSMo;!|h+9IhZza)1zo&SrmyR zJd&3cl1bL^5295aAVdgbVNMw@874VTp3eZi;KTiGl;!lNh|a#}AW*QNgxxO3K^%)K zfdSpQq_9fIaTMgjn1^*^&bNZfo_TT#gDKr=oMV{lEu@tqxDh6dY-Kl6o|uyG|NDRb z-y8tsaM#yZLEG3#cou=eq&FP&*guA?u0p}GuI-{=*bCCM$FAbB{bgi+QDVRhqUsc_ z11SICkf8e!tLQ2M3o{wFiExP%fSZVm=lA3}C_%WaEkx6dAw@yBG9p*Fhqluo2m3>= zsz5Z2`ORMIqVBA53600B+Liky!Hjf;S3xpnh;h}Zh?d6g zv0TUL_O1CSoPl$!Cnb^GKs=x#B=u6Ujegb5|EkIUlbmbZ7AX(hD&fKQg6z0LFDC@g znykx{AAE965YqO8E;1EFF<%J%m$LEv>G(A_xrB&B>|l;DBaHCxiy&qs>*-*iPR{M&p@cV&k_E-NJcsz`o$)Tc6l=;pm5dN0 zix}tMl1xy;x8Pt4!~{r5kVc?^WfTT=I!JtQLOv*R0Y1TfX$kd9t=VZ#Y zraXqOjO!*KO)@e!bW@CV%BZQgHNCJrndL;Qg7G!j1h%2(qnL}le7cZ!3z;B zzq8xm4R2RlBi^tH2wheIKPB?cTx>#$|D}Es#FyOnM2pgaeQ=e8X&Ohi=ldgnpnox-9ZUFDPZ2#K&Vnd6-Yoq2$q( z;AXXflj=lQ_@T?p4C)JDAsc!cu+b*+jtw4@Mg(LBRLV_= zWC;fr_a!s!U}WEYuq@9r4e9*#nZr*Bc+OxN)9iz;GnY!(0{*G!Faz-77#MP^fcXQB z19v(cC_Iz{1eE|oX^QJ0;*6HSJiCli3(^(N_@6Q!v=?Fo+5I!js{ftUaEXT>{<&#PY&c(@bo- z|3}8LeauDhrySFr-oI;iOwQuH*3m6ky^D)qj9_rJV=c|%%zgYxM>Q)M&q8P2P#!4> zec?Q++_nB^yZ6=3WA(NW-NOo1S?9ndqlH-fvXN-V&nw#|YHQ<@PYXGE?wUtFCE@+@ z$0-Tb0nwUC_OO?dTQyiqV{)3am8UjWt61b*jU;SY`8e;?V%7 zmHMV;T41NIrnYlX!$hYlRq>a~K+DzQ;hyhOibg`gnpv!oxlH&eo7+j{w^Y(!Gk7a^ zGkLp+akk~seA@uOKtR7CWlZKNNpM2*ezu6gBW3H+qb&mfaADl;EhaY>Xn~&-8HE=& zBb+dl&WrVeVt^3iN*;{IWRXy}vfwFuTgB7N6M>lq`K1oOj3MT-5@8wd#TBP38tqFI z#rtFanlENf$_>Xz%e%?nsTayii4nIj%Ckkvt-lu`Xw-c})UbEiBIyDV8{7WpxSNKf z{v++|-wjy*b>qD4Eu?%3ZRS17Q8b?~z%C7uWUSK}1xYAnb)aD32h+>Hh;1X&TZ zJks}8*#>^E4w@z_8?9O1gB0yRy|Fjk;0N)d0E-S8Ext z;^1Q2C6jKD$f{fNQetB#+)B2}RyUFv2F#P|dr+B3>qZ^naDq-actcra9Y>&RF{Kjh zG7tNLk*BOD!orV<5pOu4v)sJ8iXI3{<1Qiz=9gtLSvr4~3E9XbVIssT+We}{GSO~b zFi7V&DMs5NrZ}6!aXus0YDl0dLofh%;&4Z?19!SWMwn8Zbr}*}Xqhb)#!&KC^d?mi z(qvDL&**ljp?KG;)JLYzPhes4u3Pm_ad2-GZiry|6u2q!`kZZWSzC_dr_9nd29 z2m#7Hww@Rfc=7`YF$40%Y%UQmkI??$`9LXLUJL&3MHS(hjY*m|%Q9MG2+axau_JfIBx8tUi^Gt?7m};%1m9b2?GTC*!7hx?Vt`a7gHL6u& z_i*VfxpMlAYMWWpHfWWaW}JidTX5DI|CLy*$<$hgv1B!N%Nt6r1I2<}VoPM3+a#_7 z+oA=aMi)-LcUgRd-o1UzBfkhQe@Kg`7%?S)lfqQnHCRD0_McRd)M!_F=!FX6DGuac zmHy*bGc%HoE)eB3B4~d8`b?$@#Yh?ki{%+{<#=<`yzBInz0JzTiBNCR1CR2vYI2^I!@}nsYzzYK7>j z4A8^JTl^;~KzN?3L2v({H}q){{7L9duV{Zt@ul-?JNJPi@VhmP)j79KWz-_qaTz&9 zGX2oce0ryv9QMZA?>YOR$ACZB?DdWxy0UEaC9*d=QkTHqT~({4uy>%9Ayt=?-3Lo| zSq)Z6a&~wxJ=eGsMn^DcT@Z72*s`!+b@yZ{01Hn=<*O1siZqX&G_j~Et7Bub#8lm$m&O@n9iOj!m<0K#Ve3$;x(Bfo+DY*dzWL8EE#Q!i~SCp zuHSNYJtEVo+=dB2F4>#I2 z6dSOuAq%ib`5jz}aIhMXV);a3BLPcWR~Ino30AF`O8q>VW0h#%mh8*|Zaf}D36Bmq zA(`?t0!icE!R*N{kGZnQsX>S`JYB>AyYk4*Alv)rp!@unf8UNspFBgOkq@^r<)WK$u}Q+k#`zXY(k+%{ zTl6Pe^qQ6&8%`m@sYB$gDXLG#BuOL#PAqp4eJ{7;Z2&PU0U{T6uu-|vM?oHq zFW5cO{emEv6`z%y%<%>LStyeRd45f@P?TXmtui?y6=OS31~M2GYXhi(vO&0+GUNvu z%`sVkO3OILU!A^v_2T&a)ytkd(%-#(EsTsHhaVQ29dp-mp;_rB^DcH*#ApShV3#KX zb|w(HA1nu)uxxBU`M{bcbyd52k;Nrj+gT{@viQi#AgvTc@7}&Pjt}IVA6mexXc4%c zYVYU-Guj(uATyFFA<&p)8RPTVxckaR6Q{vkXi~B(8L|q3u!BhVB77Pb0o z3xcKfC!8jiixD2vm}749@--o^pmL&wU@!b<5QykOCvyv-vR=}EqNtqTgMQv_3w?Rlgm6gR+b|{LvpWO zjH0+cJ#(bZ6U&YPpJ2VW1_i1e?9 zeeu!_3?UA|A4n)-M4T!Ikns$Mix^Y^emPOKK^@qWabXCX7;_rfC$A<%;G{*pijjr* zjL|$2>q{iZIyE$7ZE$0Bc0qAhHPp97dQuFeg;GTs5D#s6gboISH<9H`6(lt31{09Z z-9UKOGy@qb=hVfx3eWpA=Am7zylPx9suH&{a>B;UiWhCg{6w9cOlU{M+ESrCbX(#k z`c(??alhgQe^eDf^qRv2M(V^PWCK~E+QL~R1k2qxcN zl%ymk)0@BI8`ECMs%R&I65)M?n+2j|*EAd2MIxzpS6I%EFb#V4>i-u>a78E1HA+Xw zt|{XhY5EU2yCfInjGc6Y^GKST?$I$x*x{5rNfvpkF!1hS6L0O-K+T2PtgMWhBDG=V zT6+nJ!Or+Xi<0RH=ivD3SMNbcPZCquYtpN8oXs6oT9u+rsoLz$;yv6A{?)ZCMEFyx zd#ljC0kC1o`#p=PD0pH%Ve=N_%vMJ%J5lVAr`;e4yBer9TSi-!xe905FY%2|Z>~q8 z^Xm6S5bH1E6z;(P`ZWf_l_S*ofr#0~od%ufs~_cQh6A_=?(Ib=*c2g#>dk?kGw^6ifql zI7proot#=?aV7GA(DJ@3#k-STfX3cWNuEck5qZ%t@8nl3-3RwdbFZnz64$WH!gB0O zV`N$a%9KcgF(eC`E+{$^=ACDl8cJafS=1X*PWPSox?uA9HlES1J2^V0$7ULiR z6_K!~15-e5WiGhP90hz5=FwCp0>jES&x)oY^M52%ELgeH!5SyyYnSJ0=9f4ii6Y|% z_-&H})^G&d3X&!X3i#;J7T~HcmooD@SJ#NR34$QX1%~E95>4RRO^Go16Y*1nEDLV7 zUz2W67(ovUiSwXQ@!6~>R%Dki z8p#v>Y4a2wVddKuVK1U&+LxIf^ZpM%p1(Tl(f4$jo&`Gr{xn#*Ew^)@$@Ad@dfa<43o_b+W9asl zQF}DPA~^#jTkD}0Gd#Y87~nY#=pu(aFWk0|?K!C#?sj#Q7Ar?6E6_DIZcc;oWlt66 zxk3>S{TJq_8>5a0ef|w#_*TS07dUpdw*DUF5p60hcA6Iv19F-zv;ciZu658R^uG+{ zf@$*MB-1j?9=NWb5pPLPd!GTtsoeh;GaR#%s7KRyg*K}9|NgUQ&xX$ae|E4p-0c7R zcpjm*d;~uXmu8VI)Jyk8uqJ&OR=il;v zXDI+U2@8K)nCo@yV2uzf_Lwbh^#vV$gSMjt(!o{bG2SsCL=zVX|Hi-4ESjG!CX?tr z>hwGQl7M@hW2>Sw7~qxj)>F^l|Nd879!faFvG_+v);W9xf3sULPH6Yw*|TTQzTSU6 z+AO)$Q%iTM#S0qT{e96dM+mAema)ZaMWKyY#XMhh5ZZV-0(&ib~flrX(bcK`a+lcIgig;*8N89?3I^VWniTEx6yzU z3Y!R)f2^?EpSKDVaMB=uBhun3d*1XoQ%G9re*Z1symz3(Db-rPJE zB$-ei>E0)VqHX+pVZ&A6iaOFOzb{|+uZBUK&VpfQM|7kWKsk17a3MU`{0e6;$UF@) zj5>e*X4u;s^akBQ$2U5xy#hk^WR>IFX#T{V)a`I7+;_pgQ>)X@bxjB90gnl8UtbX3neJI0FeK)$`* za@47yL=^?M5D(`pn04SU3ov>r=7EU7(;0Ji60QturEh0=9M1i~#Dg7hVGbpCLl&mDE7PyM&$X z_aDRF@PO^TMzn{)@tF6J{D==uk&2cU@?9-W$9g5#;xmt~qWKX|nIFZN-lX`O`67oL z0!W%mZL zbpy61YZerxAY?#uUu_Me_M*9;uKaQI58StFLf~0>x>H z$2+~PlZhk)9Eeaf=X`2J40dY!4TH%uLtbFZJ;SAaI=5qvBm@~qPwRKJ4JYXTv0fT- z7G?MeXq~%m<-8eLLWezrC@CsDaTs#r5U$$Xe+EbdV|H!j76*J(A^E-Xzr_Rxq0Cjg6>-kO3h+5@KiI*q1Zxog{$?p zHG*2g*ZsG=<3Qu>HiGFPMg^G{`c{mmOFkrVHptL1x5IeOHpbplb2X3&s-Ekn)I_#g zy-3o6+O>Y&G^ry;b^oyaOi9c;hBUNp{r7fUTIZ5I^XPC~WR%(jwHaU|JK{Ji9pHN9 zwkvt7@2ig^gz;-PV4bcR-=+EJX%#x~Q6l4O%CoDsA3RZ?M`!J6;2nEyppGcWnB>c^ zfn{JUuJ4Gmb!2OqZOg~|bDeLq6NQm03}OjAGmR&kj-DAHsz{HFy>9|vZU6H=oXto^ zaiY%fwf4l%ux&5h9odx~wVK@43Z3M=6dawwDv1=~ut|g53bVpX!9A?kwAWB4!xPOTzH6aP%6Sfp@09lb-k7u1nk#8{)@H0}%_-)VX~K=dA+)Z#y0cX^_z_nH)7WRS<+hU%u-q=}Cl&>egK} zY50hKUl0mCo#8B^QWy8QFm^^smk@$kG?y80nTHT(_zKjWK4O8J*EVk-Rj75Dx<%MA zw&j`Zq${ODLk}lp*X)T~ht{oXGEqYeJp5a4r$x(&SgIe2`dx)XNBq%Q7H%moeBTqM z($)VYven%d?qThDvj`Cfs^ z_D%D;XK-zkV={{?PLIVrH=Lk~I%}UIi!a`3gu-&u)EHu$iK4~q5=+jL$!<2Brt&H) zoP9+myXWVL*D@(h0hSs&LC66S#!-tfx!oy`Z9uqOBot{_=8Kp{hHQ4O)EGEUq1kAN%hxRrw5$Pfw79)DUBl zq_7}}R&=tLs2U`=SJu4m%@(hE5(5g0>tMiOs2K*&H5{fyr3Y zA<`qv1_(?rchEL$qBRdRUp-~J?CWKA*-izG;R>r|NK6`zRwFN5*H%GO_J-@DDY~%+ z+gTMZ-OBbaJ)Ar`lFIBcUStpq)>z0_?MB(?o$%j*qr~#R(cTo=Ys2_#fUtcxj4~t+ zZBGs3TY|66Z~Nvqc?25Y5A2n2871LKjyh2S3Ftbyj>S8;CR-NPt(Jfo4*xoU&X^5b zOTa40hQumqPrZNXsLnMY{(8MIZm`6LnsJ3V0XqxuK8H{yIt`e#&uVpS_|3s3Y@S}T z%>q+cm94JgXKR5ia-6}cE?Z{Jr{CTq+%-x$@o=KD2R670XNjD~qKaHP^QD8V-O)5@ zgZMwn^z1J?J!hB}^1Ygo47p59_%asf!2aIj03g$4u6Z9&xc7{#t8AZJNmxI8@H(XN zibmp8_QA@dT3J!2G-dD21gG`YoHU~fKq38TJt#K^sDf2Y4q5`tzB!*0G{Z5(6G{Ub ziR}Rm5<0_~xcUlX)kThx>B@P!h+`ffCGQDo5{{sn44{F?1bq+zE>z)LG!N1oZC`%P z7U)fkcXko6>j)!Bv^^&omZ7m`IG&3Ny)BN%(U!>dW}w2}$rz`)(V_S4-qse$wrYX7 zU`zl_Sz-sUVFCf(mOfWmqc}28bjI+~n-u>z0VZ}Hr2pgt_6uZCoH#^S8XQEp zxqG3 z#3l0wyF#TryK%L8v8-@6DZka<-pbGvmGK`;<(c)PcN5H;M}(-3|FE}zxW8}5f7sg} z96sH|f4Gn5Lxm35*luPJC%sg9Jl`_}8@L*7@t{toCFqtJ(Mb``IzlW|)I&D>-(aFm5!V=)hySWp&sHe#TKYqoH}Mx=Je-n-p80 z=S!=^!T13=Md6Uu=iOs7f`%FcZ|n`$btQam-d0bw;EHP60HMLORr!1t@b-9CTre$s zY+%wi90{9sLgv#&x3@LM8Z~yOn7VI59ZCjc5q~l7VAIeyKvR?5)eNZ#paahc| z!L4Jnst76j*`<&!vOTTVR6Yw@H>@iA*)}tIXk*AKtUf;+%!sg!VHmYU=L&{A?T#>WM;8bXjL&p?O$Za-88to4jmP)HzcZb3U& zv;CbT&knd&J*a3<_h?572n{k?feHGs&H+$nS9K#%D7dyEDWcGJY>K$8GDz2W#2dQC zqxOrI)xN9MD$`z;41@UG9_R9w?y4tm0y>`I?2Kjs#nYRz9y!I<(KEN@_4H5&>JTTC z%7Nb&V-0Cdm|1-DmXmt8vl{=s1U^o>7Im81Ib4=FRV}FN89SM1nd8WREJ^kxC=Y2{ z4!4wqZ-lH^aC-Vh`|&fO)s3AI{je6OnIf&}mX*ahO;8!gmL@(1yw>DH)~y>8BS2ld zF%kLwgks9gj3;vwEs=B$p{||Ps`(9|G_xI2rc~a%R!HZyDkO4~PZxcFIATTTa(0K? zwKBKu%4BWmpC*#O-1;FJMM~kUqD=Ya))G1_Kjl2jWc8Hu6q8HL`8AHFGsylsP{YL9 zkh6)ixj7?uxkM@!N!>EJi-qzDmWsTETM7G0Bnwu=z14&ZW@x}AsGeDUJqKxNUOqWI zWoi4Cz`d-crah^&k}2Bo)ax{h<~hn*W^FW@7rVP09_TuMLtUxaqJyX_u~@C zdcZG>@q}b!{0cOXFF5o!Bow6A|E&eS<-nIkcOE5gaS+~|;W0@}F8|-~V9&|_ zx3~A~U~gmpc^^-u{in|A+5=IVG$DS>m%$Ih23-c_GAnogO=Rb66;l#%hZ!ejD7P!+ zi1;l^rGcS|SZ$XQ#T2h*RN8b1S2FA@160APt<%(1z;HwtD)hH4gIK{ZR*HL7X01zn zvkPulY@szRU37cP<0vu+AJORP0Zztl48_lsbg$)UPXDF7Q#;!Kr+WJTbnvu5 z{|CvD z9zG)KZM!&hCQ`bEQ82#5NqA;ARM`;|sgK@pl`A5*T$9a3?HJlb*&#Eagh}&8bi! z16-DK;!Zs}N3FR(YR&c4*5Lm;GGq5`xkR8Ejx|-b<^CeZ`B6h84cl#lQDkpxNsCbx zp)O)S`5H-M0pYR99C9@qnEh(yBco17cH1drhvjt3QW$ojxNagvkL?1MXOliROUaN^ zSytd%JuN9^4Yhv6G3RP3OBay&2&&@NH3^UB6fJ<^^5h~Egp$R4gfokToeC@GBD&g` z&V%C*tuQj$Ds9{u&kv=sc8b__H>+v0n#{(%G_hRZd^S=ZD0{-5GZIL$Sd1k~LB51&38IQRd9;r_GD{r^6mmG1wNn&IrNSQ1Ud1)kz< z1qTKQp9n!;<#|l@GHAt1t8@otw2+$e?)p2o{{k%#<`tTE>kGW+Dnb71C@&G?Hx`dP zc5iFl0(DyXrW;W_=clOP%c5QM7zSvO&?uH`3(mvh-LPEU0<=M#c@iQ4+2KhAPTo(;M+)&B1)Fk&+CkwGq7+cYl#Gm=+7{Z=@gx zfkAO)A;=>SDFg5d0?!_=Fraa5@5os%lqi5d#wFcpguPetMYWuJ z{(>*bJctsTLAP&W$h?h7%&Z%k4b^i$L(T4JXw`cfYJ6Aoj<9}8?rgPd#(YZeZs=|Z zl`{+_7e3VH(uXQ9e#mqAyPNJbkv&vwp{~29N#-tE8s}$PLtggL&!BP8G%jq;qD#d3 z_o|;tHNu#Wtn9RplK8-+1#GwK7E8lMijsmVqIlOg@aFfwl!Py%JX@sf9Qw;5oHn)t zsdiOhI-h2+3Uba?s0HueC4qyS^{yp)9(Wdlx#i;pzFn`a%Z=ZS%Q1rzMU7W0 zhCjRm3#AtHKTo?EiE(=apgQ{hY;fS({|pa@8~T4APgDB8ZMaF>v7XXA)ul{0qF6Qx z#xgUZ?^Bwm+@@Vbq>{v;VL}c_d(l57;hFH6=UGBJ`t*29Vr-p7&}^`vGm=IBf@`%- zX|g$2%FY#WG5;Xbbeuy(cq3UyeTu2?BI?UBWkE9HfXSuc=37C--te$DfZ+Q@Odv>i zkVfBSWRd1a=$}vcT6yw|72C@xqq8E(ayqJ@41NKiLK&L@phlRLhhU1M{0Qj%Kc6ge z3@>F(i08t?qp!G@)^Dxap8k&_$Q{w*7^vm{2YYV(=flB<{@=^0wq;{SGaZ z(=57*Vm!s+5t;;X&d!BLA_eZ4WLQKpK=dZXdCv^Fz|{*+R3ObK@(rG+^yX!hsTXNR z*b#w4^@dC5)z6SzC)Yt19-p4*N^Gs<6eslW(Bg|Yi016;jO6qOp8ry#%wC?HzOY}M zp1jmAvPE*7zss=@BRxW=ID;HtbcS_Q6GRCs$=#GQLZ}{~90fVL#&O&O0PF^Kgr2-* zV2_iVCm^VL9VLtRvVrqil#7*)f;@+K6Rfd>gbY}Q(IStMDLPKm<7`eclm(E4iq3)r zoxK8B6sIYi<4RO%_65zoR24{PL;e&&TkMx-Z{wJ&$>EgaC$1*(>zL!kPh@T z58fZ4r-z68hg%3ua?WhrXPX%g!XK0P<}D%g+bG67vyPnMhj^a1eVU*DOSyi-Q*-|B z4-5LA;nQaaoAduZo)2aY+zJL@2W=;$sqfb!$FJT;IgJt#f>|8w{jWHMnnW>1QO-`o zIKm;ip5X-cCy!kgk%WDn;dqXsB&R_f1A?ps&qFA5ttccohj96rHD}~VcRbX$OI+0J zUGn_Gc#BgZ86TVNn2^gwiaJ-r4(jlZ*Ee8Z5_U|ycXl9ObBV1X?8OXozUpKG!<{3V zE%1)Z`84vNHwSDDZekugrw*IXUOwy32)XPWp)|u?F}u6)W06j?AjHMOT^O*EvF$7^juKTe;U8zs7&dQ|X>O@7Li=Up5ppHDaq&jv2&3G2^gUpV^t9 zOrHE?#Du`T_)2{7bvR0uxX}l1EFg6_;LdVbjho?92fJ$JG-$$LA?jMIZKz2rjYBKQ zH4Cf{T0O49XUiphxlbGZkK4DeJ^sVto+tjp;l}^tUY<{&&a0F&mGdsJ`o46zJhvl^ zCvrcQ+qH=8_(tXc$MHn`ZIj`R}*K;jH;25&(o zR7A)8>NsGJqcE#=^Qvx1nn437*S@USr=%oYnQV-4!l?5?u9c2G-RD^_nM7kQ!LA=< zZ?(c520ReV1o2D7do>fWH|XIHBm`@B6;3`8FY0?GOM0yNgquJuh}5vN=W>Q-P3y9% zy^K&YCh-wEe{pJzVTReV!CChjkG2fEcGtq?n_+KnP-WpF&YDxz&WzI37o33i{OiHj zgF?3~WExRk9bK6%juv#^t*Ho&8$Q!Dy=-h}xn?WNx}nHMA#JxRqiUC%HG`>kF}=%qTu@I|8Ouk zaOHpd2OIg{{XFZS|2`R%a|r0=)fpp=`|Mb5=xIxpyeBj}H6WR{w9|gOe~NJZ~5TmZ%g))QDjiG`5q!r!?JbELAq@ zAS2pg4LcxfM^?TTgreFMEj0iu-xj-fjYXwTvpQj}l^4KjMMsc!e zezurQqW7rNFClNm@L^WW>;4?mEE?w>8N(@y#&0o@_9&Sek&qjXNdbb{iCQ~^T42WL z>}ZdN!W4Q~VPOa^RT-fcfpg;=LDSJcFD?R}W9#DL7b$WAfh3eTl6sSiC{&a?je{}R zoLXzKrs4KTn}l*{v>7R12640T<+n^$To}fy*4rlYFAXoAT6fZpFL@;oI3%K|=_4+6 z;_ZrXHmy#L>H|oHmtPI0f5MvLs*aWF6RH)i{BQm7pZA|Vb>qJr4EHwjzk7MsA^)og=)B(e%N$0X zX^s;``Tk-M7k0?{Vi0#51tE62%80vdJ&FFX!GzZpM%Y@34I=y+;f2eqHi$9oi|1SYQ4_SzKegW}WFm;E|HBTg0VXPl!qzb;+zYpz=+#V=`63%~$e&_&u96 zoaOy7$#6GJ@};~$>g<0G3h_VohliW^ulMqJ_&+w+FO$5$`pNgoj@bFGm%pvf&F|H@ zgJBsa`RXcL4G5v%$Vke1#oOg|xE6Z^Re4karF1n;?H@>jx2$8rV~y?1ixLA`l(V2b zbZ65k?iet8$aNVYmu}lM+YxBHm1SYSZE%0qV00vx6tDuR0WLd|O z^uLuAP!0V*I4H#bJ2-s0iT`vj&+_!&wh>t27`T7iAWhmGa-=I7e*&W&O%sywNS>w{ zNM1?N-AnJS$bdZtmQhj3MEmTCBo!^QDRKnLtA>u!=WzY`C(CLy+F3IB!PXfu zRiAH$rJOqEcrC92zpl11*sg-Y$WE~yyDF0%YGYDmt=Sf4R2^d@bs88$HQ<7p&5cy> zRv9hTRBfirFT=QrsZvc{fBOZsXa6e5`q?58v7Pe1eS@_Eor?6jXwj647pk-NX8$c_phmCHt_%!7IffxsA1Uink&H%Oi|Gw-0^>pxb zBmcRV$HV`#xjs&#Gnnbmk-ZPnh%eJSVeo}P#)D?=-4U?ZdwYH#9+TgmRj=w%kzsc@ z$+dAs&Qhx!=0>LRtsqqMnw6j$ah$Cnc!LnkC{1(AW>CLr#!*({;N>ezxQCX2)H+?4 zWppzpt&Y3AXZ~^>xEn0yoiLe|6T2j<*%)SFnK&nu=>H( z3D`KRhI_iWUW!oyDQ)oCWt9FMXVK)Q5vXD{4$>>5sT@OFQqxdI&@~QXErKcP3@TeL zQ6Ol&z^7)71>vkv|Cw1|37c!?%HKV8 zP{Q4%gSO)+r>NJWPfc-eG|Vu*J688BW&cJr_D6bFJpUV+gVmk?&xVEbfB1BNbN=7Q zv*z>vVcUj%5*|R`hwhUJSkjhki357o<9W5i`HMc98(86e5#82(+9{FgR5PfyAimNd zqZ*NF^TUR9YmE_&7;ROA!)u)=t-h6;!ixObdd~l9Jz&lG|8#$_@0|b74xerGfA{j- z&H4Ws+Xj~J$Um;FVD*?3l#8{?;sGcbOQqDc$ggTw*rbt`R)&r6ZfJAZ4%U5IAXc4~ z578pAH7~u)tTt98w}y0A#)HfBvayD`F{DbZIsUyiY+pOZ8uEogcD{Dlf`T0F?1(^w z63wFi{4P|BcQqJE6fZFpMRj9gSp2F?R>Uu@UY3p)Rj3tO*rnikmZkBt(paKZ+ih)e zmjt?Y%iN9qL~HDu+fUR^@CtSmtzd5qc<=scbN|09|L>>!Zv3D9;o(O9e=kqT{om*R z_kd*hYjN1+8Yy_)c&UkTJNYt*NwZ$Gh_7BH z!&@#GL+f?3f|PL+7@+af=KhE2IBe$eV?TBBzrAAo$EOE-oBQ8=JSF$PS9BbDFM#61 zgZA=a>vT=mC;`e4+psqx3dTe6-f`w+-3~hdzQ#w#E8;UN7He#rP9DP3n*J}L0IH+^ zhlj5GZ-0NV$^U&X&rFOTtEDBs z)mpvA#$UXRy}nutEsgWL5A|SMmj1U<0M*(54~7F*|MP6H$^UaN&noo)bGPyroB0o} zDKbW(pepjs_CwYc`S4wbvd9Cs*u_?-)e`EW%If#7FH$GrgHaf@WQ7_|P-D?=lgjs~ zNb+{SsV1qaa-HRT?ULmQUprvD-o~W8@s4U`pWPHh6WehgEN+Oet4YKB257id%TTBX zgfr@3A>K5@qfmX>!MMaXHX!k$5gLWcng{2<8w4`*Z&H6CvlY~(zGjxOwQmMzqayvB zRHVK+T|w7X2X3S6+B_9coBKa-081zUYxRHoh5P?-u(AKYm#5_Zf6AuXd$E^a9<&A! zR_dDW#}bxny3OA|5EY<1E9=$)a`?+A0j-)FE#O0Y+SC8`3cyv|KZtirgM*flZACOlUdpuas3oq%ta)~y%NZPe%n zKFBElszyb2fMKJW#k>H|z1oNd!7Ft)DnsK};$DYi<*C(w{kFH5GEBhsHYiC#~3mQUX=Gy6)( z{l?EqwA9fQTdvpBRMcsmX2ssPpRLAbz0XSLe?uKm&G~=$?CCRC|MP5<|K)z3^_>5o zyB^3S4S#T5kUJCweNgG-KV+RyDbnlE3zY$1LN`>3XAAvMJ&b$T5!pxYL)8;C;*r}- zT4Q}tV+i-CGb$gArh21V2z9PxP4bS^^Uo#8S4y$_jFPU@cix+9rDD0=Pm+}i64lCi zuN53y^|!1du(Zi#lz}aPZ4`o^lR~g`VpmWKHiFq!G1%(B{lfike{i_D|J}#qTf1zUc_3cvo+gSWgraSB9XrWUCrV%$bjktPYqc!DZ$%Y- z>G|;rXIT_t31bpOF+UyKRlg2$l;JtK!eI}c0`G|yIYz}r?x`FlImJQP>y&S<&&iFY z#r}^mT`K)U&HmpzI2^k6zXyjK{r~+uVlQq7Nr<-D);4!#jHx65JLap@ULzG zeuK~O?~5qIA?myc5(Y5Gw65+qz@c8xa8lO%H^9VqP{+6JB>p9JTS%j<5h(4bJx==o zO-sslb{>XME5`Irq*)FN&PGOkHAFfOR>Gynr6Ag3eObRytGlXHYdNV@u`@+^RO3;g zG#Jm`h?U9f+V76qjfx|X@o)qlEDU2&Gpu_qPwSFvdNbgSWi<`z)*HimfHW4XG^o22 z0(>c*M{`UTR7*XKmrYRdLUHi93&BaHRAOy%vKdK8Chwf}gJBexeTP?Ucyb!d)8~8CKpVl)+y3cBQt|L56`na=K}Jyntz}jS547=dcopSR57OosNCP@M zLVYQ|7zZ5C=7@NZN5n%Sw4dj3)owqPrg3VQ8>UqY>!Suuk_)xRxDBqY&`vWl!b@E5$0M6SB5z+yDaH)Eqp4PI1T8mW8WV( z1a3p%{&)mVZiu@hapf(nPUIx)^^!Rs)P{u46;JyTL~ba~hT>FEoE0T(EAQ8pvu%Dc zG9PQhZUj7=&3spzc_SuSkCX^FoG%Y&^pb1zPb(U(()pNf_q!fmfLUxJZ(0H+^|3FwRR=>)YhEspN96o z2?=pm{%m3T4^N%_|NdSv{>#&i{qMazrS`vU#$SmEFE;}I!CR_@C9n^G#~j#)O{q=r zrykSK>8aWOGz+?9^cxh}MzkpFjZxXJ&1KM&idR^&LC^EW{fO>oWw#&x^h z0&}o~9*bSR*>j)uQ@Q`ASuhEbpg#uP+B`_RVUW*8B*?-xMnH}PNY zotF{WOs-$;S94PGfSTOUncM@g$0Ij*h-Ty>HoH zz3*Z&LLI%zy2IY^p#N3xx11y$0mIaaH%HTIa`Z~(#Rz&d}E;uqu=HoP=vt174 z1xYAQC?KJs0&)rsu%MaQ-TP|i=br(`#~ltFbsm4% zy9JE9$UzYy2gxnCK^krp%Izo3qJ&OR=il<~zvZ3UkO0~X=<$w={zpW!MKVTD_u97J}6}Xp!(kFaTz7B2J(7Wg{*TtMj?G)p)8%&Ji^yI5E8_dVTjwyF5 z&&#TI_6K>V<%TFA_DV-5K}PK3Q-y(*}Z!n#a=4(b*(SQ-6(Ohi^PA#2H zVw7#M(5S6(l`Xki+m;~8)s2;`z@k#g+=QK*plZ-zV#q_v$4Z8NO#fGf_ z-m@T@7@=bH^kv(i{2gc*r919&qdCrSz;=b)DQ|HaN8=!Oju|cASR^!x8EevnLT3Wn zPB|6r+Id21sA;3apOe$BXxwSQq;08%MIEIwx;f4a_CUZlzIREAvw-68==tFJz(cGa zFG;Oxh!ZHSN{LJp+E6c7xxbCLDiV!$;>wLQ*ow6#OqG0fj59h9l3;2C%yxFuu~Ej_ zM?vvJ8@|M9ds!yM2C^*2QVwO^?2FaXY7C48syPc|RWpCA$i{g1jvxtR{2d?nH_%83 ziz#1jV#?u}+b^1@F~pjHEwo%qHjLIgO^04$19D~_bd?t;f4#M3%=@yQM)E&bA7jpP zmt3(;gRjfQfz|TA!)FHr*Z$|}{^3Ude=iSP`u(rAzQYOUhR`GKKe;%7WteOoUHgeM)(fDMk4E~x7MFLaV1l7%=! zl*o^C7UgIX#TZSZET<^LL5L&bDOi>&qmq%N&tGt_T-^OGCbRH?~Z3d zM)N++Fz(NTDCvtA-Xp0$3&xjQU-coe%m=iWQ_&BUN!Cx+RpL(3TH3KJPk68I)DC#5s2QPJLq_N zR4o4=&wI~%&%4k4rJHJ39?06P~KQIalb-Z?`5+)}T9 z*)>ZD8_$X|#?t5<8Q|e#9Ocxm3f0xGTLTadBg(dLN8F)}Ptyd`ldyAyl0_Vw@3SDC zof8t%DD51fq48-Jg<$%TkvR82j6BfTkxfVj)MFJiAl!iseY!V(DKdUO?e4IroTcE7gc#&mn-{hDgGT}QWM>!=K4pEdKI>Xjgz6$gt zier1(d5e6P1!05}wh^j^VVXtZDOq7+U`_8;6o~N@Ct+ci7Rm*2G)<24bN_DiSGk%t zEGZ_SVd;7BzG=xQX;?CgrZajLg-!dG&&YMtqEU!{ib7cPHg)k9r-TM^L(o@nN_A`j zWAUk4AaRrc1ez_XwizpSEjm%PwmtOHeSWE^&c^4ILp z1F>|yV|1k5zpfkG>9}Ltwko!5blkCR+qP}nM#o9Vwte!x>wnHVd+)W!sHzY3e5o<2 z>KXU_yRJEB$m*~b!wGDjL&TmxTr`@X@?iOxy$AD%PP7MooNqcM(qfRBpWtg(>m3s_FxY$!S z;5ln#Xhkk3Eq$r+mFb}sl9{PK6>IO;JIjdp-d$Zu6az7Owze4ru{+SUYFT@-COhjz z$z)lK-C=_}FcTS3$1PZC%U2Ytw^ryf*j9Yl3LJmAw0Sg}a>@*uET^YE(T#h%RBUl| zzt3lt23Ab`^4}`-wM-10Q~5#X-|AWYJz|H3bUiAk>p|QS6HV@xw`#=HyVcqJ+nWKa zI73!uYT*1gcoC|cBNr#Kl}m&=nXul5;!=sVk-g8O6aBWBQbH)c-OnG}j@gPuvy&t3 zlhijtd0 zL$S{STI-N)o+Ur?f^eL0zxL;&G!m7Vru<)qzBk~>Q;2MV>^hop@sUL;?>Wg z3w{p~qA*(u8YN2hL?KRH8Rf~?Qxl}nM)k^1E~Ss?wvP|?nR64)*WHh`r?^IQf`LPs zOA9VSkMW?r0Dw?ewIT*pp1l%)XpK{=QUhtln&HphOe?z8=y+L8Hk0RiaScB5gK8f_ zANG5R+p-(%C+J|=JD=6pB*AxPsdT&NljPcB6BQ|@NK+l8wC6tD<_SKp6dISh@mNvf znzmp14N|yFxdqsOLyj4jlWO|KO%IBl+d$=3#iPm96u*feVX|Kmk6I8|aPr>Wy9#ef zT!n*6e60xmaV)S!2T=(ZsGSW6fdS{;(`0%RoDA?&iO+VnESwuRZ^2uWJUK&;4V7{J zxeO)wAi1F6CM7wI!5C$cJJzHQYxPhB6{=^4H+pp3P%-e8lm@_zXxb-J>QM(@*J@t* z#c+Up_DOa_r}Gf6+l-B%T7lQNBJX)F2}5h|IjwLVG0x6yv&k*1y}ewwZ6UBZVuMOy z^|*G_z3gw{6~AT^Lr&rUZuBg4fxOLCM6h7)39YFNS)0m7O7=Km z#TBwOv1|9&tLw^#H}h=^&wU%=qvi?nbDnQU$86@!8FP*02*ZV@fU}#SW!nPtO~K<- zY%1nv%kQgcX1D2X^3B=j?_;}T-g|^^f7=4d{zDr@Oy#UYbV>#RcWI1w;!Z_WJuE{m zCA5QeVu#{w0C4BVz9$9W#DR}E@vz5Er3TE(_&cIUfLX}EfY9gifXwHUhuioq|0veh zzURumr)w%F{$w_O$s88BIN-5rhR!-*!y!W%{`>J7`Y}*w#xSVu5}Tbz^6tyN1)OOS zImDA=;Q06|2Iv=93^eo%G7@Y@EUn9&a3z=~UJ4iQ}XBIGwFL1Q>5fVZ3|6aTm zj{KAn56%y)P$rbZlEl!Ew9h1tA7tmnr?-J-lvH+)2L2Eh+-yvC;lC}bMgrAtuq8DB z;<}<1DPRvjVaJb&EM=kguWw3)Ts0A|_A4Rg8q= zU-&`1r_VER#4}^p^)i<_Nb)Ad(4G0c--5Ef@pA0#7$E43R+cN~F`FMB|M|za4&t%B zwb2~)kk#`+S-Wvxw%J6Ym~1t1XY7K)-8IR!+4#F-!DV4ih=e5Xu!_Wd#OdDi(FICZ zM3F%J)KKbnXVM1u)ECSzAmEOheSfrb3<}>_ZaO;d5AM%`Z1>2;$Cq3x8P%7Hxw9Qq zZM+x~*f&k4RX~ZVacdl!F>Pp~Z5Fq&!5iKX@aHI2fn5kRnv)}k{R+!`sc@hDkpdtZ zsc9}O9M1EqfU=v^TqHEA_F@}31~biq?2y-%b(l|h(Aj_N9)CVCn}vY%QnECD0$bXS zL4zfZHdfMt6HU!jft#?J$T~JHJ{ZIs{}&XzR@cB5{LOng2f7C4pF`LWvS4Al zQ4zA*MEf!6QtUjd)&2aU5R0KnY}gLECi3MuHUg=gG5KmzA-dnxc3M86u`lQ(_)Lip zL*|O}OTyFf2mr`*`nkU9&H$o*c)2`XaryJE+Y=CJ8P6G+#Jvr&50I?nd! znPKHa(y@52ixrx%xzh?gajols2Ud^D?c{3N5o48NJsr_+L4rq8`TqtPQJ4?~n3gR-@8_J+z(-V`6`61|jAr+#DwU2B2%Vqn}1 zl-hT2{y{3R?kpAkp-P60p7crx~cgMBfZU@{ifRoog8 z;uM2|T4Ii*7q1zMu=^+Ua1pNYUV};ybqny@xdn147pFm!Ls~AF$frLF(z*xwcw3~g zTDY)zd3d)_h~-?a0`(e;vdX3(RiGzgWOFaAQe_L%^0PIEwk=VV!JUQY5*x6@Fv^-) zrm?mhz3f;W=GQz>rq%A8e;WxJiyX% z$cT6^=IgTqad~9BvkQXUQ~DJy>zawC3qF{_u0H@(*U=9ZFY`KPKQP6*{cj+>M{hL8 zj&KrG=k>cg3;nxX9(Mb2I3%R!CkP}D$KMn#EW#k~BG(yGyztXOqJbSq(n630G1UGT zxmnzR^K7FJ$B7#_jz4{M-FALPUeF`%Sd8T@PPbk}=SC&#HZJB_yb5xSCq0jHdO445 zy6v7@LkWjE@9iFLWvWT}x<(itQys4Jyk|P_cr_6phdQ1W&OMh8Y+!Ua!LH)Jk1v1P zjGqmKj1MW?=j4eI!5~8$2J^xzDSk@PX7WIubVwkc=sGcQIW9cM-TqB;N`&T^f9^q$K2iy| z_+rt&7t>*1D;00vC;rl9Vp5hWh@6774l}oc`d%XMoKxt!@h}I82f#Up>)b@wrfD` z^^hG11WGckSPc}$?jXtOYlzI~YsjI>O&V-8UJ=&PpVDcEar?qtkU$>n={X+oT^qes@6UoF;orMblfz0>{V<4Ac?@ge+XUeTT@j5yx6i;zB$T*NKR=jRXiXLL zwD11?v47wfEz~?v#JD@kX9kYiBh80i!UIGkwZ3*3Jwx^2V27plDv@}{0~Va+_@JEK z1u+KD3O)z?2?Di{1qCnx5tsmNdoCDIB2X#Gc9{XVdY2991^Eg8;nSBJdmLf&{#gKB#3xe%jO>39yBSi zB*jw;Nv}hrqE(TTK=EXN(_h9)(56Ku2E=_j21KhGHv#7#r_-sB@4a0248$QW(Jpsl z0w7F-!8>Bz7a`kONHrzs0Z0u>>t-u|+T=b=ZetV~zH(9q44$72hzxzWxpr+H3lA7! z>V^?sh7zL5WARD(1Gj;leIcBIMmhxfgMWtT;y}DYGnXj>oBAp;F@P4zZTn*Nu1+zW zSUd9ky#kn&ydI+!6yV9T_DN^HrRvG4oaI8HrqZ<^8+JL-<#D4!mzp(GV)S zI)BUm#wuJ9<5I}Z(bVyF5a}m80JIO3sK9Tstg!+RuxYPk8@2@0xv0;E#8Xx#|A&J{ZTTP3&vT1d4^)BOYk9 zh({zb{q|QR2>%a8AX0}wAkyz0JE+xAgd6QiACwR&d`FK!h-QFAKtjQgpb6^t1!qu2 zWHWaLZ!&#@D&eC6Ck@%$0`FudKLkSEgaktKVUY|;Z<@Jr{`myEK|zTV!?5^J0MVNV z$~rsGQ{lY)xZQ@gQX_~*8zy=s%xs!*MYKrB7Sn!1$3cROx!&n6UMeSX^ED5toP*rSQ5W3xmF=bX~?^MQ+19rb&fV}2U4eG z6(0(-aQ|KA=yS;Mi;bp6qEnSr)UV=1zWJh8c=lT~yq|J<*m4BHK9YKkXVmFbonA!U zw2eE|UOPH6o^K*(N)hKkl4`DM%(qlSwnKG==vby|8y39iE*HuhtiNMAcyFwQ9iOCK zpG0WAohS?5PwXm6Zm(-|^D&JhK@*B72797U6sDEgN2tG~(YzN0Yqotth<+y&!MEaz zD?p0ykEjy|^p*YQ!aEy2X>(@7-nnTti)>VA9rt}IaFF*ZNGxAkYZt>Zp^Kia@ ze;)LQ`z*&ovu63KTfU=f;d2$XkZ%|WjBj8I*6F|njo(JBI@XO_57$%JYS#b&_ha1s z-oz%jC(=e1%i$wJc4WZi0@`Fj|Ht#-0&WI!Mw8RX>7;_@#P5M4PF?cA&8infTu;9K z0OraEm5)k(qOR)lARnxBF0Qk3_a3>%yG!)}T-bF-Y0u<(18S22YNleg1EF|_?Y^V{ zs}Mr3@C#_cMVmIVKVy2cE^PKZ0j{B(V=-2eUMz=4C`vPCqr@u$Mk6d)+!k`N5*HZ} z0sBe?Gc?{Am+GqeO3vG9C<_)RO6QC{13W@p=*7?_rnTDfnpQC>^`5KfJ;8B@!##`* zHB)aMIT&QBtqM}aDDk3Hll}QZ(z~k)7EnLQoA|)?7(W6NlTo_R)|&@ft7dvQTwMhw zU(mEnae;Dvh4B$v;0&6UV(=za?0J+ef9%J5Eoo_DNj*i9V zQyw#D?}QB6Yb+d!i5)Y)65C;>x$v&mzwpBT#uyERHok|y4f=|JienCbxbDLo?vs&@xd9W<)?4Ke<{)2Y_f3feQ04j0?qPbH*a z@~kXR5826)-snR-*P*WXW6tPLRkgv}>g%A~_QUr~*X3rS=nG z!swA;h?@st{#pdeney+yr-22ktvRamkt@4!%yHmWd?|rcrUCh_(a#c&+kXP=n$KRC zW7yE!ZjqSh2wZ;_Z`n}#yp&WeTXed1WO5>aC`Ck~II9AgEDb&;Q9kHy{6$n;k=~}5 zN?CEGn2MzyAikepf3E~Zkj3bZS`Yq8X^4`i(@8~RkB2~+@DV3ZzrKGa#veZTTcy!R zG|{N)c$lSJVZ3+gCXzLJV;q7Z+*?dTM@YO5bBz?6&5fi$1ZXPd9_K_bh}G!qbZBuY zyr@F9u^cYbGMl1ld!UY@00;-o+g;I0`PE3+jzrN3=qE4-h+U#05QmTlM09G1^>#ko zEOKsvy`v+p12m{F7!YMwHZ2j5Z(^4!qu9ays_KunMxQ)-`$ZEw+=v_3wAj$g-Z@g|nWE zCCIy1<;lCP=fiB5H^Y?ADwiTSA7HQ#m{jqQ$#A*Zr2HYjYhwN%YAv^91Tf^|jiT=h z1W6bns}P;hwK1P0*~EbUCG*u~Ua|O^!KY_iXtrmiadRT@E}F1EjHC_5BE_j|X@(UA z*B%a>Q%N|MXXZcJjkt7q#L&2svsb)RD&YR04iL%0->k@440>QcIJ61+5>9#B@$;rvd z&O9tI^IRHJ#SQP+FJ?C?wrEc{!F}OJv%gHc7JbNR6P+!Eoqo;GJ{}s^=pux0l~G*r zA6;qk^_LjfrM-CUR5K`FU=@4bjM_o}Qs$Dwd%#8gQZg<1(P^Vh$)8 z-kX}C=+5dzp(WO}5Gop8Dan#o&zG7iU2J@56k^xNw*m9gm$Z~EThwkPeWRHC)xepA z{kPfjTbB<4DQ-Sol#J5UYSuQe$(X8Ilx|Fptco($^U_a+j+qtz-mnY0ZQJrdg9L3` z(vYVCd|OgHyLwA2fni&z50SL!CjKl=*1t*}Rm&Nv(n^%h4ao`%S?Ct?CJ~^DOh6jX zpd>kvW>F_3V6TEF?V(Cp-F(oviXBs>ra>U4t6@M;v~`qVb*4fx8mFd8CSI1pMW*IX zrkSN=Z~5$wx0kBzRxBd<=bdG9?P#dKt)m{2mCr?uD~j9U`cf8?V?np(YIck3h_|}n zuh46)W_DPjX7x1ra(DK1#$e4KI!ksRkv%1!v-7-p3k1qlJiJ#eOFvfr$Cfw{0t_c8 z6_pV2>MBkGbzL?>>so8ERaGFQ!0clyeaz{@5fFB^2fw-8NaKnqh$N8;P59|P3m{d{ zT5R7X0y2mSzrg!84ft%XaIy=wRCxR4J0_e(X)o{zK`bnFgkF1w5^sxA*yI56rk^w$ zn^g-3=v2f|dx~uC?HkkhDipOCiNV!SG*}*vsym(C!!3(u(Jo)l8;#>`KMzWN@l=3v zChI|MzQPJ}M4s`*2@72-L)wo*J=>5Gx}NWGuzC0cqWm-S(Q0@6Guw^UMd-_d5KGK+ z-Bq(SOUx6Kh&HQ*S(+(Q;HPI@Uzu5Ub0c7au%;(R*Z9=k`j%hGO~nCJMt_k|dfhPN zRiKXyj47MndIf+eDu5GhKnts(EBZhMfPF!Lz=6imffGv}sX&Fs`1VeaC3m;MLqU*8 zf3Y$^p>PxRWX!5Fr*a5{JR1Gi8(dQ?mt z6nzA*e5d4^y7h^l7@&mfCdN`hR#H~q;`W~J1vas+3HA5a2R-0SXyck?u)>hZo!kpA z-W3VO3$1vSQ~|dnu2Z}KqBtWDst8$#AdVZ`p87f#`?{beD94Q;&+Br*t!(SsNS6D+ z#3XQbvH)BKs=58SV-@}fw+(WV*YLCodeXICxN-h7V2Rx2qGPE9GuN-02mx$CE*eRiKOa__ zA8k z`E!MhqZrt;mAXfXMMe|q6_vg!%4E-(SBmT;OBhdpC&b}&OR z5~49=?jhz^GUu!W1|Wd}Vo~a=nEdT59K1PNEaWmnq4N8acFd0%6@4GWMbRtcbn` z0#+#2(|$4)2!;TUn5X4me0~NpbPNT)SSE4D3X>G^+5T9hQt+lf7z$|E(lZn;;|^95 zIq_bv&k#p5&e|td6v+~00+3q@TeFRj*+~7icl4t??Qvs}YlOY{3B1LWI>>zve9ofL zPG_sK-K(8u(gnhLx3kFI9b(YQ6?l> z3li1km~h}%WbFg>xsi4Bxq;UuNMB^(EtS^C`ZqVrac}?J{j~ik5=HtHZFG$YD|R7|Uh?6|3&y0Cy?!)8jE#{lE@XvIuYz!R@GaZ$lCq2B@-VRT2FZ zN27|DMi7Z(?vCe5(-QF{n$0SXz2ur&i zO->zSzLre1Oxw2BR3|OhDJbu`@5o43eA)eb-A>wTVUcij+@v$as3k;IH#ZI zY514cU50*x*j^epWWnUxOt1W8oAe$gR1-K+k}Kt}bCj{6vsPF-e-qA1G2SCELct;- zaQca@p-d{@m2rg7=&M|~%?Xq}9)gr?Kl;n=F9?#*L-BN|ST(<{l<0+MIb#WJNf4>- zyNs8p)KU%Z1*s_Gi!^AJ>LVOD`CKXh_Zr=6fEB$zph-wfsB;T!0I6 z-ez(G{{)-FopS4y7dgjU~ z)VZV-Dsv(h#p*B0EH@XZD$=w(eI3-D6ilBzXghIF{mTx2%(s^u3H z+lg%3?8!Pk^3i+R6BRxkTO-(hkcvuJ$1);FJ14=Mo~7a)@OU~n?DH$#{XEUZ&nH2& zkduehp&Jz={mUpe?&|42KVs3sKR*JN6LpU|a2)k^`l6q^i^??0)+Q4b8i!-|?>K=y z5W`M1CB)iGvQzb^pU{N_Ia?@752~ECS@%nGWAmT08q-~Y=R@}CJ z(z%z@zc_}dh|Gq$cJzXJbko-MhN@63?HNYd)vsl@&Bpmp{FUa9>5UZq(Jvj0~Dl(l;yWb`lv`DA9#+ zKH+L}jyhw`m#kzXiClco%(C9Wz08P5m_tKS6W&LQ7sZS$P*{qyXJW-_{f47|ak=Dy zF_n2a{3eJhwK$130cSIL%A#bfO6D8FyW^fDE;KT*aM{HPqPM*XO`ZO6#ty0HByWih zU6ew&${J%Crh|XHixy5wWua$?5mPFGMe5D{BZE=l4JUn`eAjhHLP8lSd|Pw`Y8`^h zu@f9Q+kEc(kdTpuF|^+@N8WT>F|&Dcm0G(xT*(o<{)s(3Fh#;z)@9*;BhXL2TxRha zA*=jS+wFbaGEU#eTyVpe)+Z9vwuk1aKIOriVoV#p2M*6w7{{vIt$EGA{pfH#q{**p zJ@k=-g?~8w_LbMf586u_r)ySvtxZvI*qVPRof;N@>9_UTp-zj2eJSEakWL7IyuS^F z97IDe@#iBx1aFR(m3z)vd_{^Jxkpvhl)SgFT1VK!6Z_izy93O4q-K@obgf!U`Wjhk zjg*n6d#{YZ@($bGGn{Z;KTPb_a13qG0kD34X53v}>8RO`O&}Cinl+#xK?pmjFP@5t zo;AA{`^&&|<(@Bp?PItc<$-aLYqEePN18sRc@u$HzaqUsfqh!5;f;dw=ZDpJ&l{nX zkBz8TmL*_-uxZm*Vxx8-PwW$_R4o#7Y~J`;XqU4pPaP$oo&%`UH#;dzCbjniH|Scs zBu)a$ZgdLzCLANs2vBzxR02Jo$9ivK#;@q@3JCKeaB&A)yRp?vf(l`{0-PK~j*%a{7OPa<y~s6VCA{XRj`g7`@Ts>@WM@zI-5cd6z% z{EL(6S)fDS`QqjN1AkQ|VMmLXX&!*+P>j<+%1l1<2W8_FZRoCwQg7=PQ>rzekpE@B zaZOSvJ4r>_QtJAKoCjo6@^MF7&@Zuhm3oFZ&pGE!?!G$M_$WIOhq&kC+^zebUhVyG zJ#vbug-nKHgAuh$PWx2lD=OL}W=2#E52jC%*7Rve{a5J2=E8%1`%@S*bXG%U^a{GAe|ZrP52>gh`2S({Ac=ob0WyPFAal_k#519HPnP>I4S1O zR<791!s&yIA0>LrWCwx7`JHRMpNYXCRu&Pv^v{H<{gqagE|;sjlU>lT>VTZcX6>Lysci1k-a5>BFD8$9I36e(_P!C-_{8JnS%>%V5(c z?w5|(=Ctq}M%Ri>tLp(CWSIW!(Tk(}QP3dc9?veQHFke1emJU<*7$1K+swrmMgY{n zgMI|2DGO^T`qZrT6vhQpVv!Wp$QZ4(6*`)+!#*vOc!zsBVk$I-lpUMkoh$z>VPrZc zKeD-?JaOB#b{q}R$xGZHz8Ktwur(&=C|GzkpH1KS5YWgZl;kR!H0fQZ@AcB%@a7oJfCbHx&X?u2#?&K0gx03W>Dv ztWGfVG7mmK`4?~`5=Ds@O-?<8YN+D4nR!sDL+|z<9708{05MOt#VPrpci3WsW)l5_ z!*I|e?MBxgo6a4Bm9-y$fXfm&%b<7av-Ne>CUS0`CUW?a$EmyYwJiiW>zbWKUA~&K zi>vF!M;zIUrMt$yPi9Qy`_~Vbo-*$kE;OgUDt_7lRx+`(5K63Pjv0<@45tr`LV z`9DH8+wo9&RX!7!O5h=Xa^FwV!aAboXa{tGcOF6k&+lIgQjS!Ij!3cAPf0{0Y#k)& zFKwh^M-CuEkikfL-&96M0cq%yEdlAvc3HdPONg=(_kxmWHi|juTGR{1^G?q%W|((* zBH-IeWRTk*I(FG&q&LposRjbT4t&IulxOhzvb~u%;Hk`H5R;VOgnW|ocQERR0|U}Gh=7Jf=S8(N=EAZ6iV5LCecc44zk-6Nmn6+Q?9KbJ zR+H|dm@~f9Wbc;Dp$W`Df+%Nxphct^D&Rd?lwUy$xjPPdf|ar0(OA%#n(QG>o-b@! z4q`yU3Nq4jcjC0`NIg(Sf?!jk4U8_9ozakD0ctkMD*Yr~rJ7;jIA9Vc_MAbos7v7Y zMywhsj#wB-!M_nbdNLtKwlpkgT(Rw0o`;DrZ&rRBFOI?NFaRA2{2wiEg~cb#tU~*C z-?eZ~gpGBLgc-m5*=X@l=WZPpF{UnRosev~Zka8f^|Ga^*O&zWbrE(FS-VQchL<)d z(2({m3xuAavSc_7@Lyb;j%65$->g;o-Z)%F=xIGtc3Ps%NVYcYDpWL(UEMP%Mj?Q@ z9Mn%V82xx^o5V=jc|hq@nmVhy)oxbB%oh21Yjm4d6hVgE90j1aQu#P`p28 z=Q{OYIc&ItZI)I(QdMcY)fI-fY|KVUe0PPlV1|fTRkcrlD^PN*#`E{vp} zs~YMypTf{05p}>k+SXEJXaDVg#phM^Q}}RgfZ<3+ z<%1ai6P}X|e|~Wli*X9;cQbf{k;_9)6~TuO$u!IKW+{zpq?C$W#EiDqAV3uIt9k^) zt>NRi*KiX6z6aTCK|EAFoZPOi=0lpQhcg&P=r@ zeE>D&DuVFqk9Ac4(oK7r%7mWWE3181h#Mq_c>LAGS956@|KY_utXn9y*%d~aqv$nM z_{}X)U zzk_es06nssV`Qy$uJDLf>=rZ~?G`?Fwrq8%CD)U~n0Fw8m*FPL)qI`yIdb9qhe|8M zx`mVqm|mm!S5Ls>H&%lmTujoMSpOkPzU@SzYPcQz6U8UPe@(1m71;ULdp@jV1@4CY zdMi{*4r5m5z@?ukCPLiM5b&;9xkC&&W|c3`M3_u&KTe00t8=MBsc`K~vn(#Z{h0`^(xCbL5{$+HaxuPBGO4`?W`k7=Cj>ruYfC5@S+ zocB%s}(TaksG72v`9+W&UY5l7R7#hIu*ezyusKOo3Apc za9Qaf+>$Kf<+MNlQ;DQiw^;fGTIPHY3%(2z7o?N2mT)#|PXg;ZAzwYzma{_G=EblW z*#=q_tjGR4>({pe$$KCH*(GjZyY;UE3H^OO#J?%H|GALbg7H+|#VX@CG8B%XO?5er zy}=(<=ifA(@n31!HR`q&D>2IAU)wY2P?;K@Vn-bnw+B`YJElxYLH zCfgWY)i^eT0SV(6ChXBgfiJ-bhX_tvd2*dvyYM9|_~~FcgApIG4k=@msutUtL{w*; zqePPh?)eS$k39zWCh!(H$n5U~PM~K*7-f`bd}Y8dW@?h?Z^oV$a?-oZG*N|zT4JiX zPAY98d+>iq=Q8colhkmKOnNY9O_lJksHNv{DXPF7-Zp&Sv7?NAi%iFa}^o0pxJ{6L*qN`T)o zTQM3O#Z}0GKKom^v-K@!qO7cRY_HxO__+IMIH|g?Iak9bAuJs(=D`mbP_j$cS*J#| z61JCcL&&kbu6EaDalhRyr=}0OUfpP7Cn}75zN3aUkF?|zb9iRoyl`i%A56{guRo(J zF>he z>ur;G5JF=OQCCrrU5{L%v@|xp8Ddp_ArZTl{)2*J5MK~v(V0X5I9&GlH2*leF&8}7 zTwu;OUv&Q=f?auXLDl!T_?eTW$Q>#jY9XpLkK5u4huKS0K!w<1HgATl6WH56V_Ll4-Hs+S$#mJW0Y zbGY`a_e0a{zIj^ud+)dvUfUk{BryxlDHjAHkaKVMl`cCqWtTz)(%2j}``f>$Beb?* zb$$QfJnf}$ohyk+6x0|Tvcdt>rCqQD5iH775=l)z2}AOqZ#?=l?Mn}NjN!fGvn4+9 zgiU)1x9CEMXIAUK+ajXJD==+OV#8~*#4|A~xYsVT+!CIH>M44wCEH5)z0?pmTdbb^hz>WSmV2ZF>y!+h5K^w=HdOYQmXT;*AwH1o z_K1fUsAE6JubP(!PYKtH{0^~Dj-KQ9DFU=WJt=`k0yKyt#Ie{GuI03_0Gpi7!2ZSTWxRXzH3AwrpHAh&_}RdIa3c$RnHUGMLHP^Z&TM&ST5&p^ zGsMN7ZG;Wy5U8NN5ncukh8J~8Oaxyj`lX&pdT`}Baa?S`m^@DTM zOWU0oW|0y63#8407h}7Xe0@K{)m%)GZVj=3J-d88hr8cGs8q5mIjB z$)2oJcc60YJKXQO*!BpL@5QEIsb^1L7Vw&S$t1|0FMm3{7dbWO=`f+WIWn2xTXdlJ zH6{{+EGWi##z;&36UBCq4sR->)<8QHc0%sk3ImlGWj8|jWJuO zh&nsLCVZArgGyAvGM1` z;>5+jPgab$%Fs(8u=aY5m+~=0?MBT!L%b-C;FP=2(a1}x=}09k1?FP+Gvt|LkhE3c zUOY;MA$~9|X$&m~Nl8`AA}7hQKJLEBmvQM?xE3!mgnoQ4o51El6}99sy-*dn&~X2L z<{!v+?Ad2^B3vQI=#cm3Ti_?c%Q6f5VM&*Me77d?KDFt!uE(%NggVP&@T|pkH<#f- z0`5+`8(%@?^xqfX1=bkPDTL5otq4kNbwz zdhQ-*gqIz^(kdQVZtGW89k`|JjiyaXdkuLeM{Y&LzR=p$m^IhR0g8~7sb*`3i1(OZ zZq@FYlHpEUbaj@o&pys{(b4ng4<$cJz(4dy64ZkoEBYV(5{9{Yv+-Z>Q_KEe;0FVq zGhD)p>_^P5+qk}A7;AK#oWx?5=t?5kyf`D>F6_R{q`$y_{Z zB|5k?km7pG>7J>Mm9D2^s_9U}SuEF+4c4QN!$0_6+p=5b*tb9PS*++jjc&=dRiX6S zxI<)9u9$&UK2*i2h#n>LKV8pq2r)dHrnBSu;U?j`>?s#J2Dvo(8^2X?j4l!lOj2pq zjqFxug?Er${%#@frj0)sPrJSh(|kWwM0j1;VdFa3F%T>s<}0pHF4P^lH)ormlnCob zA5T0-1a^_5aCg}t)>LC3E(u`7f`8>2=7oQ%V>@+{ixm9!EX(_yn)rHn|8037Tz5yX ztY-KP@l*jrJ(nwIy-mUpCV7-)yAmdfQv= zET?;2P^ux0Jf4;`ukXeUMbk-BKU>AQ_(Npuo;$ zNH@(tMel?1UpnmiT+N$jg&hf!6Z3tMN;2B|}YGuW;T& zmsBzFk^aTYL3S9ut=;VO|89#}|F%V&usWu8*C0dAn6;FZAz}+fK-l)D83WOO>SD%V ze*%h+5>fF9rolSW>8pw4`^t@@y{Ps=TaGQ4*zqj+AKDu_+=(GP!SQXqp!&S3W@XW9 z(TMvmZga%7fIkA#I&EiNQ6AhGxcVu>7j&~#&UV7qdmy5OIRBv`6pr6Cq_oFj1{;z$ ziT+o~{f@`~)(mG2-~h2ing80C2)MHo#5R4?v~=KJ2uVU(VkPA7au!}t!$z;rl(eMo zjLp^|2Gf@qkcfz(oIQpUl>seGv~y-6*51cn^zKw4c;E8ntkL2BB46JBZ}R2-{~=#4 z>G%JKi1?T!V9AmEOGE~+%KrMAmkyY|NQh%P7)X6=BCftTP@kS?qfJa6otjqumuOkn zvM&Quh5lU(DJ+`CV>L*`3caHSuL<&U+l}tT*}T1HjRO&vj24#~jTSGSV&YU`LQq3L zxdxuU0(3J5jf$L0Mwe+@wo@1LBRh6Yx?N$Jy8 z0&=DHE)v6JFkRq|E==RzF1ehHO^2CaGPo3G-c~(ml!*-QZcf7fTs=X2<3O4Od&U@l zPNX$2DJleL3j)@-wV#D%h)?ukt~lDvdtDmzMA#SPgk)sibq4o&R^iZEeHt#QhuqVeIJ zzi(9hLsS61x1%%hNHlmi8u5yxu=o4G*%8E)r+6tW{%XpAF42WduzArtwRs%6V-(4A zWNpB+$6scyZaE3i!^E{DUP-IuQDMq5u}A^(uX5y}lH--d3XEz)WUG`Ao6$m4j3jT- zA1YbbFlTv!P3u49@C5eWgyY}&S5cvioJ6}slE(_EQu zjOjMeJSwP#IxPk*r)W#{lw#Hr>~Wcviu?U+G+lqYBq7L?-C|Inwjo>uGbuc337hQx zObmO@5J+25HR(sJ91zKaaw!*&xDF|U&n*@rq~(0JO7w3_a@Aw;cjL?VyddifV5!!d8y(7r3}bFTY1T< zjIL9$=C7nvUeYND^%C@`(`kY7>Y7LZO-(?2g%6QoLr^@~4*Z;Zr3lMIc?y(*Oq2yF ze6CzKS6FCW`TxV&J4Q$Lwe7wY+qP}nwrxA<*z9!NvD2|_vxAOp+h)f(^?#n{ynFBW z-RH}xF)~JKRn{7{s#fNj^SZC!byp-VluMFUh&E8eiWhWu5X$I=7n!Ttr4Z>PWhZH& z-QrjV3oQ_Q$Ill75?lnqln3R{*GNq!73Nt6h6e-VckVKlRyq4$+h`_1O<{E*I5$7m~~ zwpXdllgJjuT!f&vy}-KSzyvR;53)F-1c7=iX;PTZ7tkJX~duyNaUmAcgQu-K$Jx)dUh7j(N$m_dFZ~d;_|(DOpr6g!Z~<&j(Zv37%kQRplya6=wYWTasaJI zfV7X=lG2w`D7Py^E#}Z%USj9)cV{XsspZVgv-8zz#8_{VFH6AG)1}sK$9AM+(tJ^$6D{!7Z~|Ve@%$7h zD8OOn=O`phZuEF0=P!cf;T|BKyH;tU()rlQX8RE$5G6%vBSFEB$Ff-%RS3H?Ta|N4 zwn(>$a9Y-dxfZ0>pq}l;gKye+P!?W_Dj#XF_aa-$p5sv|p}q)g==)1{Eu?O(sBUai zr={&q^kY-QmTs9ozYA+9Q4Y!~+v0nYsMD8h1-fO#9%_FnS;6b^hbI?BbI6MVHB#5) zvg%`?)6_@mmDgZIbggEWnC_R>dCmjY+1FEt^4DMIn#9-M44hcGua@;}7LLeVlQ4Xu z_bWb5^j-fF2`{w%c$^x0p^qp6`l+m4o3e2@ckgIY$9!@p8xg|<=ia==)YNSUWe+V) z$JVY1$Njqwg|V}-=y6MaCMXDomZ090Womy=Ts}fmwGc)zlzX5L?30O?8tNY@HQ4Uv zgO2xd;rOARHmd1KNAE)qFQ03L3ex3bBkAZGd$Ul5dFb!u!}l@Rjj8c<+s)5WD$7%T zjaBS8h&bBo^m%yaweSmf@-uhx6L<2%OJN2X5PD@3B{vwK-!;C=ExByozse2N&<#}I z6QqS}M^CL@x7;U)G$cJzfGplA+nOM|Ke1sAQf~8j)nVpNLcpYpXB;3AO*0<7^N)no z%efjFi?nyiH-f5wjpt}@Ab`5!l8s?&oi?n$)^;Cv(p?=H;edKELAKNig_Hi>d zimc%0s>gS#2r>q&rJG;lz)Zk$dR-;dLagg=aGwQp#$8cOeGaDv;qP-Fd|#k)cG zSQe0w%+gmCks?ouD2!Wr;iMa264=3{T^~{3h8PnB7fsZDk1qR`8-{0&Rq%Q|$0!d; z;?38;;1SAsA{ssEs~^t_yqYX<7qBbEQ_fWq@t4dKbn1QW znvc4Q;=Rg^Rp7LhJYw>q$Kdi?Wm9#=8pdSTpZV5Wo0hj55|-87ZiCROeN@A)x@Rwe z@E4z59~}3p$F42f=&_Bq2H1q^F!ih@tTEH)4O;ecu{TxK+}DjO}C(C9Kyx z!NPW8F+2D&3{~q(lx&O;LFObFBRzdq4^-OJ&h3Fp9JN0eAP-1ifn`={iitk6u7!|C z_X3##&x@22L+kKm(Wb|3x9x%nyfD7?%Xa5)HR|ouiwh{SXp!S^$^DcGEA*9|!=2*5 zI58uX_yCH>*p6G;PKhV2QnNL1Ae*L&I{o%WXqORC2AfkNMi$x-aK(@zI>%*Cf1T7N z_Nde{I(Rp$N?gCIc8bbOT|y#sj*r9P@b83wnuY@A||Cj#HD3effMm9Mez}tORxR z#;ZOb;PiY>2-+Uk_4b!}GSymZFPX-ecW1{!fOG@6^v&Wyf^B#ETzG*^7R@P@W2A(z z6$vLjlJ8VLLuTd0?+)X~Hy4sS2K-d~ky6ZE@k3zsp8bZ7q7*+)Ixn2^Qlg++<>@fO zwyUdBbAsY5KEcY0m5NgxtCdW4!Cx#X=R#DHb1mCC6q=(B>-$SZh4R$R;RN@rh(54?ggLOxW)-cuKX04iEbvkkYOQ2PgpU)=&Jt@u( zSZDSVfdKtC`fr!CGP+qlHSo(ajsnNTeJ5k=55V+K-m* zJS~1s=nLdweE6YUd*QD{=X4ML6Ha-yT1T+!(K5j^n#Y|f7_IdXq!Ei7%6I#Zos`?# zoL`*il^1Gb{WKvyP|DZF>%K4GXRji=?z?+PTK3$SnY=`BL(8Ru{rCPIzPw+Nn^JUB zeOr@V5Ccj4->Jj02{yzjY2o~h2VKN=?2b9?hl&L0>NdnOR9l&cf#w0W_bpfY{t|Lz zd3LLJ&;)goI0={85Lt&!c=)?DdbL-1{ynt9e?Z=FiEpS4O2(|ZbApD8vM^1fHcsBV zk@0s@!d<>o7ZzeD3Gaz9lDEq|an^9X@*`9#XkTF+LcT}gzIseAMdFl}g5~nh^z>kC zm$q+}ds!6pyJ&24z7nd;1FXCJsrO^!yU1B3zm(2M=AeaDvl_Iqj_g@u25(JvFx?|p zS!B^83Obad*&qspEGDk^pqK_ynu;6p$EVN9p3_OAhaccRCrJJjPf+O$uZajQoLzE2S9aF)oVU8RZlH6(Z1?O z*g*Njz@7*Z<*Stn)yO63`yZLxFw4WQO{RA}x-D*%XmkjQ3ccr6OKJ?Vhth%Ft9*>=bW+xE6`L-31 z3T*tmtVvAqtU39->fQfsgH8RHd%c+|+pCl^fQqm>HUUxXAFOt2vVHqbn*8ED)e%yV z%n(@GFS#M8sHiCICj6?csCfB$Qh#jny%RmpepKS9UOx9UD%+gD8vVcFwLJgEYf(vQ z!G45*`_uf|Whx{pjcNasDBa)fd z4Zb-a{HdC~5_EybI7hwWXGl+D!amNR*n~`gx!PGF3kMAD|6jVc74(0jYel=@M{C-% zsWeAshv?A9X%>1083|RASY_XzgubIhMy_qoBnzUrz$wYXn^Fg05s=yAoy^=~f|J$X zj*$Q9|ES+TJ&0;6aTUVK{2#gxXupi*>e=wvzqs?!r4@RL(VFPPu?ZPe=GHJ1qwho zZ$JU)N?&7K{;Yt3WN4{<-StfgayFT7>K|REmEZ`0G$H?-x4GRy83u|r?H>a-diw3L zVvP4NE3SJ+JL&sSdu5A~GhL{S=`vGuRhR)euz)ekr~Y>y6jNNu* zn&zVh*F?H~rdmkhn!uR2M%t>CT0}2k7p!<@%AlOaqaU57pME1n5isOM)m!J??ZMTy z1928ca(1SNbXyBV1DN)OglMB3m45G^9~MC4N>XxZ+If?ykK@r;C*RJ?Mq0_$t4^Vn z|0*9ojJuNe&+ZZY1>qqPut z)G_*~trNKsE5hkWnLNv3%f_>-NKNb4HH3AJA{Y!4S)`^dh3!y>&1kijQCVS4hz=@A%>ZM95>6k)YR3>1oQ_zQg?|IV>FMf8xL2P) zh?4ti$QE`npq|yZ_2CxR`iM;D-P|T7i=5%XxNgIG6E%}lt^?I$leD^D%T63`RCq9Z zA!asf@;e`}SLB8ps2lqUpp>^rIr8{tA*gE;v6T%eIP$u1@;}_$?h0C1ir`>$ z50cKkx=4ar1prRh&wgUyl*FxbaPfMvxbb?x9a#x5;_-;NgdKsOlp{n( z_1&bzh@Ec;bKxgoVx1}-DnA|Y7jOl#QEA=a00#T*T4$hJEyx$}7xAl$?b++u#{TBG^z&7%kz&2EuR2evs5S z$}YPoa(U)BxgfGzw#oYZhU`jeE>U}+tDJcU$m3J_E{KFi6TuBvkOO_gNyzqRu?~aB z@L@sT(gG9GIQE2_g*>R+>}>|1m5e#`z^)hJUsIMgxi7t+blki34<_4cZzaJ#7}*YG zj;IZ3ppl`^*Bo(bW_hgU4MQbHs4hjVDcg}4saEAJmCrl1%dp}?8&Nn-k7BOC* z*5SrGZtO&~vDhT}J4b!d#94~NL@r>%&&MkCdFihS3YV}>cdxLS>M~`MbL`77m_bH) zIQ_SDE&u#uf%tBDPQ7et4pw$*fmB|Y1hEZd?CR<77}UI~^t~6@6LvRNBX~xtd8Bs5 z0-l~7?cJu{+5*=!rl;J^{5(87IEk_CR+xRLIuzZ0qEBfmT#|pa4H??ih<1&Yp?#r! zCa;8}o+I~Wv;;i5a4$sKAU21Mlk+-W*VfuGq+}k&A5;wJkKQ$c*@<`YcJDZQ(O~*i zJ-#{(cd`(`M{A1qU_?LFjEWy>G71Xe#i^U+Aw<-mZV=@@`BLe&N;Ro=O|?{4LvgMg`In}H@m>tNjx`f=*pk zJ5up&a8#uI(7?6o%y_oafA&a}@l?X++ifyBIR5G<<8M&Ktj=W@lDyY}Vw7&nOHDefZ#O0_7oWZ!J&hkEPXN^*q@8Xo+qxmU0I=jjrYX7SP0xS?>g#)1+u7!<#S zJ$$jDN_6csE_mR^Rfg}e?lOGCFnn3QVaD(ccCo?pcx4#5r`m;<7FU@LcANXgM!8mw_+aXc3L71JlIfG2-=;5 zOJTstb?94Yb3UW`6NxgGE!5t&XUX6MVo8OtLD>a_dw$M#!H*8*Im>Bp1gp!^X*V1( z(?`a%uqgYAxGw!XUUdrK6~Mx$O>ico1)qE!A1{=|0R~ZEUQ1ttUxuO~@+q%5iMSNu zlb8)TCawXzrZB)PfUkzKswh*!j)`OBwv$=%X;%~Un?WK&ZT;j~35*^v&;H?(VAV5&1kRd|cx zU~_Dg_`2m6@}1*&ImxxAOZEM?A!+J@+<%NSB2IM<;&zw?@M9ynGSHidM}0lcL`(_MiVMYb;l_R6G+z`LcH9Cm9 zOXBUC8PK~+^6Bq>omTNq!%DJ~;4N#Rn?TTGCR%thUdtdtQPclGdbyqZwKq=o&50;w zH4vCAVB#4rliuePf6}~p8o-KZR=pSuTSV$X?;1PON|+l9~C7UPdj61-Gc& z0oWnEJO&sMSGf~vDPy~{w+`qi#l(@6S4Y~f1!)lCfB(C?+uNwKX7yPMylpf$$$Zvb z7sjV_RBSX#J}9sv{Dqwyo(6iqQUcg^T!D7~=l7p_Oz(E%edKwrIFF14uD9bLR7bdr z!;w@$XdqOWgdp4~-%RBjptx@{z|TOG_3NSlj@TE1Mfgef%pmk)FAXeyf};Y7+=7-3AOlDQkpD;qkl&l@!IS@P zr?ZBZZg{@%7fsA?DFjg7hXN=iu!2TmO#w+M3!TuEKpZ{wtMw`taAk5U_Jp}O?&c%> z7<2}*2uB05cs$y|ZT{Ad-UF_ktN#Xjp2oe35bx`K4QMY_+6K6Ig0zFVY(pL}2Jis> z=oh{Q-NFON0Dt<)0L-AqF)~j8X2C4lry%SkyKNM+MzWtE?5Tedh0R{w06zh^e=*n@ zH#QkOok;(VTL^f$|HUcF1(Sw~H}nDzas!;Qj1mSt9DBfG8$TYbc~%)dToqJCG=tO~ z!+_L9GXgF!1_3K{<(?>Y(ME^>*5xw6AbbEY5Jkre93Tyd2ks3RvGQzUA|3(m4Ni0O z1R8;k2K+GqkkUnfh-<1|lSTl)hj~Mz$O)+H>#JbT?mXT!`4y2Dh(ZV-)Zpmcc>n=i+KueZUk?K)lH#+K(??)TfW zFmGa?+7a%c&oE4(=c&RWQOnAJ+c^S$P~RcbtYe4T=PRRYA^cBt#`SU@A$VJFS!P$$;Me zgCF_k71+ZOAD-DOfAT);_p;hF4CVY^kbPtB{(is@xKO|k39RJ(8e&LdeVOiPVXxgQ zNAEKs=J2%r&nu&yUs%Aokpp51CDiB@EF}~;Wq2kf6i#9A=q`%_V#1t$J)%36In}7j z#ji2q9>e00k>|4Z7qO(s!og1!j;l#{V!=+X<-*iE|1hvJ+ed<-#Ow)`tN+Uk>a>N? zGRDgz=`SU=l`O*dl|ycrYKd<7aEmL$Y75ty0Rle5dyYpwfz<=L8@NQKf(LKAjJ!-| z7n*LI3|iZf*W~w1yxXnxHt%utb~-IW4h3^=PY(ug;mfRCff`))gE^yq1lW&9cLv33 z8+w@<-lWWLy#(51dyA(M#`Ey}J`0Dc1mW?Vp^pvA=Z6SYSi9rk>u|4lA?|82U-COeF8=^JguvHfh9_NDp`FSYrJKP1+(hjZEacvrs$z-9Xq4Usm>mUz_b1dl{?v9FnfKnr*dYH>Cwq~n<9g)IEdtH|sxZM{ zY?>REG4tyrW9wVo+xJIoH=Iyq%BYV1FU(2yUzihWB$gbdzjC7N*a%fm*p_eXzdUbd z=j~Mt@zFeT@5mOohp4I2iEhFF2K`gfGcPD)Ends^+DMe7TOv>)q+ZB~xBQp9CBVFptDw#s@ZXETg@!mYWcb@>_6{~`Fj{8bN;6DZ|3Okv3r2(V@e+26 zS$l{3_;=u7ixNVCS*?R{Ag@cE)$_kX2T5dnO?k=eTOxte9>~neg>Wcw`P{tKD!fmd za4xwmd?K~`wW<-l>iTi&E-j2WnP@cHBGN%5_;8%y`PDzL()+pzad^fD*g&G&z10Eo7n`?=pB)Qf3Lz@tYwJXpNI z2jZy6|9{5{y0510haq`(DYn(N9tpYKyg(yO)h%f~HhxAtbW~t|T3DZ*R8^jR2+mG* znjv`7+t0Jy>Wn2-CsWmqq^~8z2~GJ!X(2Wcggwx-P=Tv-9%gD&1&VC1j@-iUksOf= zHPAfHre}ixEI(#DK(WuOH_MzNXr>w|dYT*(xhtQw!t7P>q^GN+>AaSYv}t~zc}m|l zAtWJyQ|mYlRZdCvy_#YJUC?&)?3dGKfJqb$2!9iu)%6TA*~P#97()Jw(R{ng4( zwytE*K=GR;{rEy$1A!vwjGc`0mKY0lKlBq`e?IkZnQd}%<(ARRw?1*;DgxDYF959rs1J(S0;815B zfxlZz(~S(5yvUs}1xW0d%M|!A{!K#_hC=@n0nm2!KlIK9qI;m>t$Kj^(o1&MKJ{io z1amjyzeuRc#E&OWp@YZi>bB`Ge`QwJ3Vr^;rC(p~oP3%QB4pzqLxEO}ykm3gZ6L6t zU%plH_WeC<$#AjUg`#mb`n^6AEceA`Y|}9AchIhs95J8w>$mux61r`BJf6EuNEy6V z&V@x483pBCWjeY87eYuc1?EVN`)z&N8?1Vg@@jGOrY5J;u6)8<@W@!7bFpgQom3$s z!D!$l(*9o1rhCEjaA2jbh_b`q|athLka3{&k{G15-^2T%PO zT1o{!)c4%R{i08G@yv48;=`kSWvmKiOj@lEngFGmKWbmx)d$HB+||qIX?D6*W9U6x zT8q@bewBHs{N<)mMi*k#3Zw2x3-kUuCh<6>f{cb+XBkoU{0`@cbrideyfSgjqUG&* zRsU0x#v_TgHf7amm`AJFrGzlIa?JV9Uu>3uKLhC-J0~LTK%(|vqGiI5SFbBtKk4k? zQ7IQV(ZAfsGvlxKmcuL|TK{H=@MG;SbbZtLrV9kcuj*I$F!9_Aj2bSInBmi)>ZebI zknZ!N4DcfzXncgva%W6Ser{vmRb~2rUU;QQS@`0yJUH!w<+^lxnshr&`}_qvXWkGB zuQ7u^zNH)e!wvK#i9n;U*3<2Nj!DzCpa!VNs)zz>zy+PZfypbOywv`zSQI8%|Is7= z_J%3>v98{$@q?bs)zgnv#lHzdUbeyWg5Wgqv5pjB`cI*r_$Gy)g zM~_EAKWGoSbiM8JnRfRKEx=Qe!0vV5fYG7QL5!V$Vbg?88lE)0ruy( z7d7{O^1zo83{=MF$YJ*6H`+$E7nghH#sjU9S4z2o8#B_G&4-tLvP+ zQ`VCpb_q_A#6FPNAJUgTb7Y41JkYxB$C3?6^ z;>em+EF@Vf6!u*=*?O0l-8JTifnnN16cP^j@8-~*D6DNi+ZIDpn}k8$ZI#I8MO=M> z$dq4?N%GAuD7hI)m0?tjVO&gxX-$z>&VEsfu<)=Uq-tkOd%H#PM$a4^hI~x*SWYB;&u;};*|{D-g|Q{zgs?MRtKqYc#fg>mvxjDJ#YOhuI6kHDR;maKYpmaPhzJkAH0 z&I7$LBQS-8RIfbG=W8{y@3ChuR$_V$24bi(z_OmN+~##Y8~G$lX@BG+zYv{AyH}n` zs)Rq@ofaS;04bX7neFh*KjFSvyGquazAwx%b0cdpex{Ayjt2UdVblRx(42qSFDsvdkzN!PK}Suw|m784|Hvgp-XeTRk*S9&BWp&Dsv< z%KDrcZHJ=7OH0d@)fvDNVMHsO+Jxnnxih9_QzH zD|B*Jgz~Az4d6+Ke-u)cK!sbzA~Imi*MAY**N1t7rA{iuOhf+6$~w6U@pp|bxj&V% zk|?yQ*!y{P{G+Guy`4@ogr1uw6He4EgQ>CdF*a$h!`Wf84R=2fGx{VgZSyRKrv`Sz!PaNRd@mwjHOGfgPM4xRj+)H?im~hV(^X^!`<9JWR`SmXwYli$ zP{RhqdE(MPk6V*o~FQxw+-!da3 z<=IX{pa+Zc;&1Zj*Zcx^tXxb!vS&J;rM@q$>S~$Mud(N^sW7YWCQJigo`M&i!Cy{z zQWT%PCvaddMDbsjL@!3C5~gdtlg^(M=@7v;NYGygVD_1pOeJWF&Yy=@$I8KqJSET* z_E9L#Ulu1vZ1h2+1X!$oK{}|f-@0kM%|;n$O;exXZCZB_CT{eZuIb{n2`9+<;xOS= zM%dfvQ5i{&>?hUgaz$*1P$utN5op)2BWhJsfuUNuz2e;=1a1W>lduLgEmoj*gIZ$I zoDDMeWl6MilTLjn&CxzFTKcZk>tv(f7+UPDDjsAVQZDmXcpZy>_Uv^tS4iJq2GGhh zbO%&OQ!d3-^6NedGz_k}7H35~&71beH6YZd8@O1FT3m^8BytyBs5D6!SkB^GXinba zQKrqyRkJqF#D-ZUYQPH^{*p3v7c}y)S^cbcqHA6-x~TU;1@O|L?yiCq$7PP;QK!T% z(d6_n=$YcpHCq%-$$1RGPe}!uSMKMcCKz3IxP&y^X{E0GWXqt?-1^4n>mvDP7m(Ha zNUA%XlR@Gu9Q~8eP~ypmrG#TuWrSmK+Hr}8i)9mh*Si<1Pe;huo5`eEgK0Lf zix{G0)&f`db9JWF@E9;te)rzvbWD6RSH<`>RM}LF-D#8s=*JbUu-Y zC{=lr2AIL5E5pQJ1$@TRJdxee%y`v`HjW!5^vL2gk6|PjP{!8(PjNeI1RLPz-gE z)bgE?Fe?u^T4U(fgB0|gw{a{hiD!0OFxWavh!s`(S}pY~^ex9xW94Y=tt(ixKS(Dl z_00<5J%lKuvTP}&7J^^wxq-U$_rCAHh+jM@jrM7DxL zxsIX;A-lfRcpo)HqZ~C?2Uj0N!ReLfT(p>}&GH@(U|2C8q zSR?ZEsFopgcem?sRPa%cCz_M-p2FFl=C)$S25@WoLL43q0VLsdt_fNC3WaPzczw^Hb_MZc4lX*mi8u zB%x}ZxF4UM85Y!7cJaA=u1jg85uUau=d{Z8E{(|WUjn>x?d$}x{SqkDoRC6J6;I9t zThn;#f-8wRWcO{K1%cBik{ibBqWs{$4Q+355uIWi0`m|zHw&a!hNWeAH#g*bHi$?{ zqa!^FH^OoOh(+m5W{+!v@SysG!IQF5l`e_s* z;simf7mcg$086rN?by|>?qNi2g`VeN?Mb0N+afHmJ#k`1@V&t@C1OSUj$%+}(<=r7 zpX^!7v6!a_mNFCm82#;_>zbz709BT&?0uRdIWP+-mtiVuh?$d1f?l#`A?3$n$!3$7 zjjD=HLxU7N^ix=xl+-8NeCettZ1wV<3U!fyGo!v6KkKQS_J*;i@4JR%6c7 zzXfB-qfF#kFuMW4qs0(H#9obJBu zIzL9C=@t_2pDcFaVOzZ}y`w+)v2-G%xOwKy44Dk>=6MzcGGI+rE$(d^1{r_E%=7FK zlv~y``!zjm@*Mto$szl&A#R|exmF%7sXE~3)vzN2J-JLIQuj9qg)NN@wF(_pI4Xzw zIcjqS8Tdj4N2_M_38HqrZauhqrAV`Gy|#9>W|5?FwdT2P^F1kwN|j~-Oyy$5qH(@c zII}+2Xhh(pt+5}%iqC%=vNLb$9Y2TEJ?> zbfvl-nv*w#fD_VFN>J;UA&{|RT#ay-9~0z&L%3n4Aa|N8)VaJMPGagst(7PtSQn z&rQ@EiP=C=MlgsM;Eo6GUO7t*Rx{qq*XOQs9dr&@zND+s<6W=Pn4P&ke)G}B+qA+L z0N^8V^Ug!;Soh5W_x|R_a*Ew&6Ev@%lz15Y70Eyec<^H zdr36v^mzr{0`_GnfLn%aV{T>J0zeKxM%J$~X?FPAiFf!{fh~8$vA-FXyP-L6&8TfOTkZx8fP{k~rpbaLDn(ynhgXVo#R_bT??XXG%sN9R zD|IEv)nQW5hYP(Q9zs#{ZQ{1iYuw#^2~rOe{M>L?{0y+qJX>jV;Jb9{(e|XY3A`U7 zumk??^*it?Gp1 zKJQ*HZ?pOkdtZ>$4oDA!w8cL+Md(S-2t3|`tp`6P$qulf)PLEk6I>bs)*za)PC;iF z^9)Glkel>iPD)0i@Brx1JC5@?5X?>ulO|?co@~ocrH#vi79(k3Mt$8lB7NNrVA|Po zp&*Z`vRo*SjM)xLxbJ$lonEptalL&Fon1*`kl3>Oa7@HKfZHGPSaA_d=rp*4SS>h4 zJ4r{VeCn6N*|6Gh>B*A(&GxXWMN*Z+ys$Dy1IALTzD5jdyR%Sc0`sbzQ_KJjL4T?K z#w6_R&(F8(-qwKEa(CD7pSN*z^IdnelMV#(G|tr;HF&1KGHPrF$A)FB8Ro}Nng{L5 zQwk7lnN8ucYPoyW21I(md-F`mqJVdOsETwDtWi9Eb(P=CQhV3hgI6iRT2445o`-)H z5x{Jox;0=xPL?if8MAP!%8Gj-z|ddMTT|GZP3y?{EBqFe-x#*>(OSn0o`D8%DLsy~ zyQ1%9BrD^F>*I8FSf@ke18}d9!JElrL=l_%-#4Y1FsC)W+qO}5KgF6eWaU)DO<1`m z*)p_ZTj9Rb7~n?hejmDM9P)$0Bx_gSLPndK29AEfJIA*ksDT81TGn679w>iB$OBh6 zReOR$IL%2T)<)`^ZF!*U%dNBTccTe$ekFn6RA8C$pX^-?z3jYSxJXlC?SJYVKmAyC zfiXJ1*Nv_I_(1He_PV^bFeSL@4ax0CuVV&@$&O*`+k0nv`fg^&1RV5eSz<6Qm&!{L z9$8l}L22t&{^#sr?4y~v*hxyJm0#i!$@l&4yUTf8q{s=>A$mq=N5|Pue0eJ-A!>qK zPpP4`?5ou)%Q@}F$Od37+#(VNBl9FxibamIjQh2U0-|LsyWPCMAtYHAjvj-}Aol{F z?##SG8#~$ME4g8gR&p}=ju_v<`|@*3@D(J>tavCvH`A=450c&Wi@k9USA3FPP4x|; zqCK!ehi1iyZIPe2RHvEV)yb$2B+Ih+Fm#~is_V!CwXE)d%CSW%C7ZFOOZc*q+8>>i z9wU@jc@^Ycs<>YnMwb>LC6BKPT!O)^(05^RSg{3EZ#5#ba(c@`f%bspSU3R#<< z<)(bWis|naei!S$Z+Kc zL}zXPa%p@cxV2fRzb3D<94|~yOzlf0N|p$HzBAVAM|7c_IP@T<-0=1gzfBH&3Jgbz zw^Pn1Gd%4EuO92fY#B2$jg7xf@O;NS5aYsjy|u2Ri;fn>)N5$(TacWUkP4oC*&4|( zi9^RObUoxH(FV9;%W z@sbfcbR1n^?c!GfZ;^yiyquCFn3S8!YC|n{x|;L;prggeKvt#tI@c(`a2X+Jxn1m- zs4_>L>oI(%$CtL9`5yZQ_F{9Y$bvQ3XtjBvc1%0=wH6~FoH3nNB95D3yq(828*`E! zFGVBSfvxtz)=yLRy~bhwmPW*>|3Yy=L@4|Uf}po${_r+x+DkVXON^iY+LOt!Z0%qV z_((L~7|7)0R?{eWN(|6wkDkFma%PM@x=&4y6j+}g7)s?Oa~yAC$X=^)RNw>si&9G_ zs85wM(BgBsMVw@Aw~A{cWIR98DcQPynS%s*@wMs5_3>pGICGsXaE`zjcHF&X%zPnq z{uDmoc#U)BxDWA6ICRx`l~kMJ-xh`+9OBs{_!_J|IFxKnjZHJNo!Wb|{mjHBOK@E= z?|kd;TQQ#@^dzuX(IOEkv0mZeVd}Tmke#Qtep$yO_AUn1i@iS%w}I!e7`>Yg|GH^- z!&JRYVD!*=_D$M{={1S)Dt%>+YOpYYv4zy2iv(?RE+f}s&C|_{Y`jD|+r&;b-mw`U ztnz^AL<(-TMp9q=@ESeAt7I{RZD8O@NRfQFv~Wp z#Od@QpN%YRRn=;x*f6{!)J?uhkHDGq{(#}n&|HSe z!cAJ4|4#z`C4}s=K|3f#73k)`8M2p|RmMkjmwjSq2LF({;(W7p1D6>R?8%(ml^R*c z2(uNW`g=iG3|krq1dbQttwbm#-dsqZTH;G?&x4+5JvvULq4I ztjn4igX_{(=bykS1xcElSY`lR6 z&dx_m>Yk_I>3W5|0-O+)VHT@M54WBg_r`GbV!P9HKU+EO7^A~{(hEtXGi9yruT-R+ z$=`e*F~jGg2Xr_5^cNNuf)3mJB(w*JkJI(!!B#@Vn+v52?!BvL?DW4pL#5);5_0<7 z&!iN*eMasx^dBgqi^=enfnLF~fi}0kKdopZnRJnVRWn2aK56O19TscZau`O_f#2X#>>Q?Eyq$pN8}!s*)dka(;I zaDpk!EK1nb+I?1g)6zqTn(moc`?4J7Dc&Ljy#d=q`iOKC8d6(Tri>{!N*Z?%+|}E9 zQ&UDSpuh95kis?WY{j@VyCJz6f@nvLX~u>{Z-V`0xG?m^^gmjHn_4(aiSWCyd`YmL3s6ef{R2uSAN(AgE(i8^e`N?7Y#k0g^18Dc zkP^5x6f_n{ApuJJ^}ygjX;MqWW9TbRrL6_~VRpM&vq#HZf*2e4L!)9lfi873Lp4Dq zy_UyCf*KM%ze?fK=G^lbT^TBv6x8I+hd=Rm5MP;yPS#puA=f%lFz!`6%!jkurKI`L z*Lo|BU~-k0u}&ENYe%u47@~gy`@zgLdAOjHdJz8W(a}s=gVZk!7-3Wa@>#5C5MS{g z;yS5=aWG%-I;n+JfqwFmOF=-e_)5tiXTHyJ$-gbWm3s3DQqqW@XoAF9?C`bX!1)QW#MHJ$k2U* zQMU)EutY>zbO3B4b;XCy(6K>mBh9HM^HZsDR?~0^ttCHtmb2K?3_`wlF=W9Z%O^(> zMGUdW8=ID5f?@PQ0L2`(J{A?X(W*nhqS}O2y2`rtM%^+NaZM&5OU=U zl=jaD(>@4DO;CMf2rMwhx)gMnw!XR6CN;)7s*>|s2ED*IY9?Gsm_HnVk@P!U0(QEU zKO15sI7nsY&V^!oP?a@seUom+=qB!-ih1#TC{xyAvNmn>k`7TdEH$GM$#l+UEf`ell&LFN!rEoKWd zWj=f$TvgYrc11BS3ZhVS%3ktip6Uur51!GN?>9PQI?rJz)DVT*UAwU)Ho`_ElrCy= z*YdC0ID~z!+vYogaa&$h88*%Na19xBQHadx1VIspr%*%RvptxWZ|n0M1*WalCQh6! z^_AWZ^IT<&%F@2JD`xghkr?tn*_*W8j$Os!Rqn7&Y@p(b)WJ^6RgRSt`AuKqx7F zzJWN1&o>=QopU$+d3^)ofY`Eo2>Vs`r?X;LMF$dridt>wZ5OV))muAKi}+P-O3b z*sWvpwlCnlXZ?4{r~lF=0^`fHS8(*MXDufd-0O7v`|hgY6`X6!HHc5V@uSw`=<=9= zu4r8E8;=3uyAa);u?!m;oYAp^tihARTR3)cocrQQ>r*%X;>iTzD}mFO^@70yyG0IO zW(fOLg_+XdZdg(g{>M9*CO)2l-UBsW%y`MdJ?pR^#rN&|(S92E2-270{&NHPs5&np z9&9Q@>P1nUrUg1Yx1VyaarU;Rt7zq{w(`(()oCV%sc+!#mc;9e5z%G%A5s?W;S}w{ zGh*)iQMy6wZ1wO1-OJU~EG}mcr&K;_9~r9_s&?~t^xp+s%m9vBLwj!(%^{O|xLK?M)D=c|7A_}!}ooBx35 zMMj7KBFi6IWh$jBn@ePCb-d!14iSfRWj?KFt3l2=$b25|p~U^q-4Gns3c1vCZhGKX zOLrYT@f)%6n@``>Kl>qu!@1(ZHFsn_!TvagHNE_ft=VdOD~;o|W{Q_P%5%KTkUbzX zU3_9Ri|RZYQ48KA#F~NZA8B!wIxayewVg{EIYDvYP`kjL#f)1yPhoulpk97KDdcA= zGwDFinlwCuCZ7wFEgz-@{t_~>yBa^!%v3Sx?rY+XzKZ?cuD4Q)v?J3lV3cZMGgfBX z>0-&))vK@9v>d*L(Q7d84A0w?v0FJS9Bit4Rq7(Hl~D15;X$Z+%g8{`YkD4qehK=x zvy<7=L8K)7=Vk~h3Qn*Z1gP3jV;eO`fte$4H);3yWL5H=FM=@^knvwfEt zQ5Fg9A~%uQFNSR(_l82tf!*F zEuh&~E`x6STsd*0=oeK%BWFy?{*PS{;?iiij4g1-V#1f7tMAHC%IsjL317lbN%?Hc znypcu@6QZmM;{+)g7Msxg1S@d49Pj!(9;lp2S&KI<5D)BBmFdc?P9{rk59VO##h_zPMOkV{8)2o{p|ARw26{ z!{e$A4H`N`xarnoI2bWVC^jg>JYvckl874**0mqkm_dx%FZ}Edxtmw^2p7JXoKuir~CVAsF^zoF>wegu+@hZ#W_(bsGKBAzan34dN>K;lP4 zG2i?Bu0`-?FUMck*2VhGlA9Dlvm*5xI}bf>8ZUft$_N0;rKI*d1!qVKeHjOB7@iTry>vKmcDAVj za3I=>=yOVe9s&vl27ngK1A+ipMr#}xBM0Rgv_1=>0dQ_e5im)P3g8tM5x}2osz9^@ z4+atixY(uJA?M`Wv<_L~YwRL6@Q*)Rw;#{I6B1tact1S7G0x>S1-Q(eHcAAFO95wWt(8*a(~hzrsY zfP;Iv1HKYBK<@DAPP#H>M}YFAbAY)JJnH$od_brMokOFXwc~gJ4r68V(nexxKVM;y za0e;`m8b=k0IVnB&w36#phu7s0P7Ko3%~=chiKSR3p39|Su`NdX?rp-ZWn|R{4RqM z>KI^3@S;P8xYMNB6uWD6Y1X3c%-^1Vysw+;7`E)v1li%~bqzRUkq$Vss>d0)KNqPz zTI-txu?DPYf#7iyZy`tTN}N-76u7+<9JqZPA>e|%A81pyI2&s75EGIr6EKjM!6<_k zk_J;k$tzMr$urfi`l;k>b%kOUmP7*PG`*j-p*Hk{&n>HtLrE zzhl1rVtAUicrci>ZR&Y8Zt&mR8Ad(Pk2^y|W+P&|qIhE9MS$S*JnD)f+3N60ld8ea zYW)@UMEIabMAsBqw-Ddc=?Cjn+DJ6`-4xOmSDgkfu@vwfBytWg!=a=qlniQqT7s^3m~aZAl3h zJemK_AR@l8fg1Q=osDs=@xvz@Sge?cCiMiQKm{ok1@Q4Q42T~N{C1?j-#V%bxRT;}!T4mTD*oLHf34Qxk?tL_s`oOrR|_20S=I7OC-P(@KuWlA&K{5$yPgvq7{_A( z)t-U^(N4opfKVcV(`*CSXf!=f&by0%b;C^3yuklfgEKk5d|MhH-R)1d*U|`@y2Rb_lZxu7$81oJBvCJ zdHQWmt^-0@z7AhNAlqGEE5}P9Xsu5i`hY?|UhK%_tFa`Z<7o)l7yQOKhX<>jF}sT? z6Xw+?i}|97t)U(omLG0l&c(f$N3VjqsYlEVPJzzNYkwc%oCE$Hdo+(}fZ)}mIF@m! zT1}oZ$sLt=QrwPRyBRSRSDGu^)MfZKXI%CglckU08uveAV<`YQJJ!^rAy<;Dco-`E zL}2kj)MZC;S>>@%mz-IL5QHt6246lPV6yyO&n8JyZ!@b#C)wGbexhOB>fwB=UcQFRFhip(xkQ= z>{?U##rfj>jLUGpy0O7=g*|V%(->qZR5)Da^Tc=@cA-0JPEvVXMYs$j6~DJyjRx+% zpmzXUx1X&n3uX;4}J&q$-qNciOMNFy5K zv9)$GVw_!eE0SfuDV#fpj78}$MUS2sW)i8QZC?RU^r+yXVpCv}e-yo_BYf%qBhsM9 z|Gz;RLNxzpq|sM)Tdl0}OWP<#c||YP1@*M0^?8z-vtt`MR|5Z40mQTUkCKNjdKT6a zdtyXvCO;>#=-0J}H17XyA&I8=Z1L&-1k?tB&a;=gB|B5mX=7CRMt%hKOyD`}AS~b# z%#0Zi(b3RK6;Mx}4J&7drssl&_5CvT8}@@#AoG|BbF>%}L=0MtOZjRyR=J7II-BC~ zl0-Xuy#~j~+s<*2xMbg7w1as?k1RiR3XKx9f`ZSjT=T)VGTlsDwT#|R)w#pPDU2#bU&HG~&QE*Hrx32#!uw|4La(hc##`Y;cKV^Qk@O|QH@#h2 zLVDh34S(F>x2~Vf4etfwL8)1scw0ga|CxC%kB0Kcvbincm``_z&R;U0WaUO-q!6vm z6z}QD0!>Z`-^DIFP7&Hmw3QTbs!PFBa>znWX6*S&JRWXpGyrZfnwh}G>2c4;(a(*W zEIHcglWz5ZAkbwJ9JXI&#h2RQ(lA+ncvj>))NS4LSP%s5o*&=D+u4L2q`_Yr=XSA%C zCaPYSSpR3<1H0RmCZ|gUYArq&Hu1uTaQuPjucign5%|!-raS2Vw6Hik_tD)@GCrwtun`o;APzz?$>#7;|Yo+VC zQrplL%tF{k`fKHE*zlKN7-FF!KREdyPEGSY|GbH+pP(Sb5KzpxQa58mP zss%6CYi#i~zI4p;AW6`7lYH-TP*S7j$k=KBeR!x`gzY;c;LLVyx{8UkZd7b0Dd+axq)3PF*_>1IA}O<|6LY&Bgaj&NrS5#?o5MEHq&$1T+Y#P$|>;-zDH0I= zxAY>Cu!>=yP^oDT!T9Zz5eTS#`Tnwj;V+?)sE2*%u>Y@8qbyuHUS9yi#h0T zfv77Hs8>F$ac~j}L^7vvkTOS@YU|_#xStV{#WhAwK|EB)a}Dgy=x&&A%nd%Q#|LA# zlix~sZ&4kdP(6>=IOM-)1@Z*;ST^@y1@x3f_At4qZc{p3*Y0@sYMdw5Vg=Q%VD&;x z@Y=0WFxx#E-Jvq~xI?%7JdLsJG%e~~3oRTuJJGwEljU8F9Dj=3Br{Cpzrp;q!|2A- zg5+iZ9l^G20z|VaIPCi1vyb3;30J_PLAbOpK32)%u; zQ!mv!?^Vq*sC?sDJ}D~}7#s7X8_7)LPvZ;Qj7}{sZQd5{Uw;vNdMvb*|bwD z-tTt7fdNTsp!>;P@pDgb7~MVFY-MSqG%+wr~SKD52x!Q$5z zjhcM9@gB&k*-6tb$e6r3BR9qUEeR$ z`r~N_p8Q`qh7`lDnGH1WkVKf?Q1jk#@_gT#PT{Y_cbK0(YluI#BnZ;+6ZcQuGm`!_ zIvK+^OcseY+7fr%K7L~NC%Ys3kUFN%r;Lr_wiYvsR`9;ae$uGyF+uSwZ}IK{%O4)g zA1;!$RJSz_JvKR@B2(;sWFDbG+-_`Lz#&O2p9nDLe~0NCDTqfTmWGlRk#asG9-TCf zt{%jzB+s^I*=c=9Zxx}kK2!WGb2LtE?3VwDVsjOWgF+ocVYCmKf|o^Py#i;N6N01_ zLD1XFHt*r!h;tJS>*XZe_vn`?$v0B~X5%4N%p6~Tr;8**6T~CR4ZliS$QrFGe77>$ zDe_^ZO1IA3XPnm`#Kt_7h|g+qry!aoLatsTX5r(45Xm|efw%csz*N-h;E|d+O1hGV zT_}}w!1Ar&Ivs7EHT*nv&;5F9FSnRceyXce4k^)CKW^Jz@ZPO$^id%_0zXyLUm%CV z3LAA>?27THTD0;}kDY&2)90X=yoC zyx_h3#*6n08Aa5C*Wdl zzQ8ZW`<`cMa}KfXGT< zNFqld48+`Ckk$d)c$};E-*yedVL!HD+@N=qsk&8#4&RW5m-3F!0lggMjPoI2NQUIL zDj$$3IqWC(`FC(K(|LDQrfM zg0TUHHaY9aikz#6an-*CHiPXhU4bGAng-C>D!e;ai;w3Yd6G|*6- zR$s(05qP|XUSW8=K?!0G?c|l?heUBbn0+^-=Z&#(eFtDY`TeZoP%~aA>!rIh4lYyb zBM77>Tg?a}M#r{L625pWvEq7bA!P!zL;eb?%hB9You`MS`=PnV^aEom|KMGw1Z@S} zUEV6(KFPFYeabgMHNK_JJ?1$~=}8l4;ZapD?YW-*V z&xGH%IpR`JKk;;e#{|pXg2#H$Y5u&(5-fWjCuGTAL#%kus`J_SX5FKDu%hiaRo_Il zfUakk%9c@vyZ}s~5CeM(L<|XW3Y3q4^v?&4*S%eza$?0AiVOEoRbd=vVh6F44GO|C zfkPK^#kL`o4*Bj&{FF_*8>ds%s21q^^7Zwwq%KfcIH6I35KhkT@GB8dW+u_k2z{!; z++N>J(>HHXS(23Q%I4`uVl&Wz{9Hm4HwPUASC7LW-?+7XA z2o}4AhvvWWe?#gu3|kYYy5R%Ag!i@fJ?q+Et$4G-2Gl*pAPIi5B3^a$v38p^X^NPj zM(v;AJR^qRi@bB9p=hd75mmFFA;OKA%QwqP;#@D7OmqIwzgZbdk?L#|x7^A5Yc$x1 zJOJ%ed~79f<2(K4;jDtR1vlQ`JxtTIDRyIjK(-x`EkZZT+io@3pL;{Sthxn!>sO*+ zHK{+)@zf2oJy=-^PbPS~9N>5UrXmpMa)t`5Pny1t+Oam_k@(C|r%bb8Dk{&hVCwL6 z=$u1)Vsoe~o9&rj+J;7Hs~g(>n6=#1)66bCsV3)BkY9Jb7+4D{;YSOg7>B3MVr z%_n&ouTm-~;ZLuyA7}glGzbE7%}C4)WL!t=Bp{9)&>opPX9^qv8dNywTc2lr^HYGr zFRtV~LNO4vhlvCDPT-#JB;4HDNs7<=_Y;PA0$!-{jaed-ZBCp1hihS4ITddf;h_ve z8Y8Z2Kk2O}B2px|$wS#PeInUndrTjZOkv-fgb#S$6GG54|0y~&6w+XKmzGPLBY*x5 zE73JIN+mq@W?D;Yc3KZ91vc@q%p4t#ro2iEjSPjla0MSuynuQl8Y~5191GXxyeoOq zt{F((_Tyj;j%JQEGI*4L5H`y~-Ajq$m9@vnnZ9gcOr=uHQj<^!c(c-B#l(5V!sudd?feozdG-!H!V?N2W0r9TD*A+&i zsN~D2Y2Mxx{K7LwEN|H&Dm+I$UQLj8PnA(e(x9-|}S`tGax-a7mor4yV*pPrw z+P7dOI;sW~Kq$=2pK0J00<0>1_CVV4h{rl z5)mqe|I8XWuulEqL{(UsE>vvO)-%|25iKcQ9uTlfRr%6mA3-Z7ei=sTGHX)~vR$Aa z>HMSOw!Dt0`fvHF@vVv>3=ginps#?l(#(P?n_}{uE0XH8P1LO;{=};odTR1Y&_Pjl zl@GJSDhSi~(pq*>!-;WLOR5Kl4hO3_3$L4&a6wcb{ZU*Nb7QB7m|bm(*R6@mxS(98hbV`aexA zRMGcV*0wL})}xl8^(mI%K4_n)Cn%u5O0SG^Ws)h78wqkXKd9KA_7rULEV9+>Q*>-1 zBhaw}Mubw|>aAO{&~A*YHIfO?55i;G`KQbF;RFulNsbCC_E)u&E-N-N4cmoeitD}( z;dM}UY?|4{H;VR)kVlO2Dc4L}%lykp0>174g&)S-|5;cWpYYDl;J#bs&BNUbQQr@A zZ40;u#LsNNuUI9cD`%g&@GDp~WVM0b8Ljx35!)uMT`gBukOpw=ja03Ql`!yW57?(& z;2puD|4`@HVLQPPge-Wzww#+#lV&-ST6C(6Xw!;9#n6oeco~L`eUm1&|AbZ|P)f9L zn`S-M)$jrAg=(AT*C&&`fS{GH6m8H5q?rcTv66Zpf3gc)`G7R#6Csi`&{oDmeXUdQ z9Y=9EG7r#c1_U_hAOy+eKn)TcRZ^H??@}-@c@4oH@MJ7xN+>7<5<-BZPzkJIe4wKc z73ev0l0w!1fhzK#I6Pvb(xK28Bd!$Dfbdq@BsiDpKQdG}Bg8*cqXS7kK>(NyJ1-GH z=a!rUq)Wu!gsMsc?~97LB?f(D3EXy9;};;3;ftrLm@bNwAqTsfr)%1CnPv7Oy7h65_Dh`RS;>w%2<3A0ge*y zE7c0ZibC+&n41yrsxahcs~eDfU%(%~U_h3pxq5sbq-`1Cp4c9H*Ab5v=ZT@pu;q)f zo&uh-?aP%4GQDNb7-#t&d~P#-b9>@h4E;wK$}#L0-Yq|kc-Xm5oQo{GPi$Qc0NA(q z{;~R7z5;CjtQ?U+YnJs-dEZD9#dw{WzIVOlxeRXOquaQ*TXa-fEHnH8_CXy4p`IXw ziUOJjr0XKGYh2|Ic|WXew0>(@GO6`q)?w9QQK9IWML2P-Kqdnx z2a=3L1;ecXeijr3tTSdkCism;4br#yI5g-ZOD8jed7P z9X10pUM;6eR4w#LVb1H->FetF#CaFZZLTfi?!7P$)6rmgR~Rr3(L??H>m>Za{p;G8 zTNs`Jbr;QIsOq#Gp&Q^4U!;1>N!0VRO$Y4?vK$!k-#_P#m95k# z+Mh9}sjGO%`Hx2iX~cKD(NmG$<@CK&n<^uskb)4bgrRsw5TJNcphRc#0rW^162PHg030f;KOHC(7{;O< zhzd?gGvXW+EzHcVx4%ImAoTWig&RYmE(#P+0Tu)=4Id~Rkn;gVnmCaW82{vae4aId zqHk{$9@t5g>41r$p`?YRL@1z%BO6z0L{WtD;h3Q#q_4Wq2$MYq2^x`nMtXRm1jM== zh@N(Yu7HCOU7-X3cVz@{S9osdl~9Gcp%NI049dxfwi^M#O~&&OkQt3Fs5}QhLY=_w zq}Wd2_Y!d6_b~*a9`gPGKL9O_@cVOXPWbO$pSuZ`Rv)?7l$d^sGy)4Z0vEumNJnrK z4V~iI1@My*A1kdu1tC29o||^zZ#trej)-a_ksvmJk|2`zBY9G#+8ZAT`#A~qmrE=r zUSSg%3H4_(06~MIV3529gc!+s1oRKZ0>S)xtgpOx7aDLD78+;(opFq0%G(71%YxlM z&^4U#>Y4oPS~BUE2A`{h+#9YF|A$(a@Qvh4@1^j-h235d%+S1(y};L(0>bzu>3190DROdA^?a=o+ z8*fJ`*_)RUiurNAk~^!te;FmA9bW)=CGW0i1HrkZ2apyinf|{>i-7(+ErPDoz>T36 z3Ci&Iu!vBjRES=$RG(CO*6--VhWHNr*9}uTm&8cVpvj;~2KIH#YS1#Y3IF6QS?u}$<}5D1es}UXPc(5ZIy1;zb#E2wovr!e+c<$<6w9G z9kZM*0Wb?%(##0hD@0c$3Cd*%pA0?pPvl8|2`7;S;pOI&r<6R-kk{9p4ga+7WqUt# z@xULaAfJrq_<1|N>FOt8rqlF^y7ILWrJFc!bx3ivMHgiG96CC&<+g}{9jqSxSI&|_ z3E(Wnu6s}NmCG0KJS~i!X}H}%>c{$B>Id=bh%0ObHZC2T7te;hAKSm(=05>Bl2^=k zi>|(*tIlOyxUhGN%^w$a?f}ff`Lr>+>y6ei-LBZJ{;-SV3?{Sm{BjIwjEt#Tkl!m? z*puqF+vC*tzxPr)l{K63_OaWx4D5XUe@K>CHl`I)I`Tsdtk`_Pz?_|lMAmZRw zeHBjqN??sX$!@#SD-N$m6)w+W`ai1%Y?o`(oe$&Fq~<8-1!e|?Clg8zZm@Z4M$P~9NAxy)X`Y~IJ& zyUKQmEH|^ymI@mE8X-f2{Q6KyiZ)HD1=sPv$(F=9#}^rN@?XOTPxjB=q86oSX9sl? zoxY?+O--)|y53d`W*+Az&Z#_oHv}Y0bS`fHZKi}CrTm+jvZ1`yeMo{VpMILwWi;!W zAjplj$*49NWx4-C^gc`uj`|OtQTLaV3$IywI{@<)_J83Zzpf4!WqL*Bjcer<5IOSI zyEdCOo<+eFkSQq~#l6mK`$I+qv;J~^9TL5aH}k=9z#u+>6BecGn;j64^we)0jTi7~VNd6=`HYHwfZIR?!MEoJVv0n2KwY7)l^6SS;U`FJFVlcU@CC5+!Mu zEmN+Pzo?vsLpZ8y}#F98m>xuBsQ zaLH)!%~crK`H-^|DE=nBa=9KYshzDjVd!>p9}z<&t(yER44w|IcqqMwQ)Kk%6Y2JV zD$7lTwNBPfO6QHw*TFF(b-k$ra!OfrN~lq5g&Bn?g$7s-LOOlDBzGNo8ZPKdzA7GJ z`jq!bel__Dm7Mc6(k6}iK)oDk)FAbv-x^CO`FO17fWw^U>uS>pQyjNu)bT%q8%A_C zWH-0$Q}>uz37%HuVy2O@9?L7IZS$6F7qx$!E%y`{1lOa`jLiipDO2z^>nbm;piq@j z1NN<14b{sj=+B;pu67qBLaR7Hj_v(8Le0^VRlsN`*j2y_S)(N?V0GtO8nD0(G<)|9 zE}j3X+3^2-f85p)d|zm96aIV}PPNkaOf@VDrY5gx8gmuZ?F(LgUIDA-eER&^LbsV! zu-vT9f5!+{y_X^b54nUg6A;DDdFnuKAVpeXe?xXiqiPo{2}bj@EcMMop^N3}l%BM) zYNNH?h-k9A7R?SG=kpYY{2n9GLqyhdZ_1#jx~Lr8E}6g6_S%o^d! z-50IkuSYLY)51)b*L=LGuls`whL)=v6`=ECTWAk?s-K5PT)jS97NHCn zQffV8=$Tk+;aDVg){vn;}lj)6G;<2_ON z@7@yuJ%9Fa>sO3T-Uzsv-^i}q91-`tA#6ZHpQ@cPr0HBTE@9PASz$WKK)grO{7^{u>%!FjKaz4P|* zuZVXOE>|caT^%^ZEy^*qHP?w*@++_mK$?xpq~P7#pl0d z2_b)@g-7dyr`dL}ZQ@S3#FQ3&{-_yy3?y&u4g78O?5CX1l5$B(wathTeDm%jOtn3o z9(MB{WD?7*xx7JLz$r%lf-hWKSL~D>dQio8?W<^PYft}4YvFy!n19&$wf-=Z2YeoQ z-pWz$lSBR&d^Ig7H~MzG-Pr_f#zAuvn+i606qOQLJN8MJO{r2RQfcS zDTuuP9L{kdIAbU|;F{uSfr5|mJh*WLu0_DA7mLnXfTH`CuH_H1mW5n*p66ZB+P9yJKL71`+NvzlFw< zik1;DZMdmUPt^cO+wRV``@=udcH)n;?Ztzc07zSqTY$6$xz)zZzywI!?#@5bmN6VJ zNMe{=|6?1n_6dm9P+90VIqFqAr%a|6N{FRFleO0qc2zZh~HCbBj@o?5#)H)G{ zq*UMKU8(F1Ikq@8F1^YmULfLa#0eL1%;TkU5J1fZ{ZVs1f7IL+K+OdVmP@1(Uy}W3^jlnIsq)Sdw(pndjOxPc5*EjQEyaNXR^{W6R9?VZoC@vx>E}>pw8ff5ffJ6pGsPiNn#D~EV-pEIA3O}#l zZQoUg=?iZp0wXkq1S6CQA%0S#*&N&ne%k{IQ;n?%-em93wXwU^t4p29BkdH3N{HLE0^oC!Xo#lf5rVa;s*~G<0mITttE=nKG9V z2B<<%$AClRzQz1U6~csGdKdl>d9qZkQfowW~*S{{(2yizuQ{&G&cM-ctSgG+4Lik;0$cC6jGwf_1Yd+^H^PbLp~rxe)av2RxHbUjU9 zpQ}eNTLlwx(KtaWp{To2Z>XBoP$jcIu!kJ}l`BLNhywrR zhwvSp2caCRt4LABCyh0K2NN^dNu%Rj52Z==%F$T>@7w(g=p3&Gq%WTwKo5&3`gmIW z(ZgVJz7WGW|5XrU;a&5HXz@7lnA=f~A~)n^rFD^|{t4gqs~BG)8Yf;|`iOBQElX3(Dga(j$B)(CJvc zpp;Kf>sV0|%U0RGhzOFDjr}lqrOvwAw zm|$SeJUioheHd1(xC-OI80T15gf@$K*6-vRt&nh&y8JSxI^&eGO!B+_kGvIzzk36t z7B?@4FWI@@@)(a5w@b*n?;U^kehkK`iho(# zD=ZiyS;#6hO-t@X_FokKaVH7Y)a1DHHUN9~E-SlnU)sikbCR~Ns@UfBd98*Wt9w&g z+adP${po0IQI{^h>$8>XOGJrk4%v%UeHqL1s{$LKL$g%X%o_Qnutb*`Zc3GAE3}+s zQx)RA)-Jl;Er6K|&lI~}SHFl3;t|V8QJnjW?yCS7zoObxRI?GkgXR?Om&@+lQr3iD z(W;5^blMbO!@M`=saUyQxZ^sC1{g@jIW18)JU`iF2EN}mYR!tG$)ouH5Wl$EZbyI5 zW>o%hzs9t$_J%@XQ74F5Y7|6;DPI)mrs}|u+?qMME>~<3byk$b6ke)aQT)WMP=@xL zGH#XJ=c-Db(6_!D8Q1k;l`%cyz?a%Y#ejTB(VZK8ATk8-;E}8Op}jGy;QJq_ZqD$R zbZ9);udx=-CV;=-(6_Vzym|X$x3}7^38luM9dxzV$oSvBft?a3(auS`WV&#g2`tmc z8_SXd+scsFQR7#C^;BShV1(Ge@ZjUaHGrcqX%bW)#>9MQ{etT9=St?&iOF-S5rRlkk8JPt3*EfIo`cEF_!Qv_-jfPg=A^$ zv2$H$`>wF5Yq)PF`Cs`A(jC|$tvV2?A4k~aE=sd5iv5fgVdJcQ<4#?42#kOTj4BEG zI)8@zFivA(d^F%T(q3qQY~&w3INRN${ocS!gzq1m`z)+u;&~wnTfU!Uid&8i zcy?}Fk5`cbf*l+HL6T-0fyvNluO*p-;+4A^1rU^V~3>Vz7^3o?)r;ZP2PV93}1 znvu(0Dr!=~%-E}A2Ug+PJ(@u-V&F`oMV@^OaZSJ{GXbd7pl%xrG$5=h;bnyAt*H*x+41nPI zjXrf_hT5lXGbtjhYT{FmlS!xfLWPs*tLUy^FMW;J9`cU%PwQs$Tr@u>5J4%Bn>2rO zdo6Dnr|)oYSIf;Ce7U`5)6z?Lq9}+Vs=eYS@(cQfk1)4*`^ScBPQ}#M6D>cU@e~cOED}D$Zyz55s{xS>I!2MFe2m|V^M6!* zO*17B5XunOXHKU|loH7GS21Q)FiZG+dEmW6&1cmS#dnz$oVoUYT80%slf}>ov+!_q zb8`}2kd2{yA3gBNlV5<{$cCF{n4W?N7WlhKObNvLF& zMWXYFRXLe7l+kI~-ueeeXy9I->44G|Jyxke6TpFHwMpn`8_z02Ui>-z1FAv#sF-@| z48Ru3pq(Td;ra~>=Ee(ZLWR~0JV>~=m&V7A z1jZ0jU90Cj9z3P&s8k;t@4-AFoJr(CpLMJ6j`_J&Mtme0?6-vb#9R7o(ql!XI;@Ul zdHA-~q*Jf6t6PZeE2d?GesRqG%P}8XI<5zNY$HU=wRwi;l&U3&BdmmR$@^ENVV*kv%(EFa` zC7|YLlcFgO)O?`hDb_^)yB<;f$@Cak?Lj5i?LM|^X(eGPq~tkb>G?1qvGG|A%eU2R z+Xt&Fr0p*4Xs$BVciD>uo_uriyBwqhAscw*cGjXs&56FK6l41Wg)PBYlMUhMYvxRe z9KRR;mB<&_rrgu8iZAAvRp&Jcxb7CX-ED#AToCR=UK;L*DZzeu=Qv|EXWDM>2h;QP zKHh>bl39;0cTbNukC^SQyOW}&7j@sPxVI(Fm-4@L)f}D+uH$Icm@yV|uFBKD?w|I& zGc!Y_YIrjehTd<#e;9gfjW77;!gp9GeRjo?)_!FB@5(i`L|?9V{#<*Es9eJ!$Cymh zSyz}z{%sCh!TU6XWuU*aRadDGH*7j{`4Hr9)phYyu>B=mbk2J=_s(&6Fin7%AQr6- zoq9*Uk-lcQ5m9Cou`z|uuclRs_nU8{=qDM0yzuX2$}ef@TVHflyO)BTKd_Vp5DxeA zB}$`*_G4A{ldccqMkC)HRm~1OS<2^b&M*4$ip@Ap(;NhFH3}>^qvNb6SYN#@DkhV0 zr7TXT(!CbQyCQ{@wt~ZU>nGBB$6tqP8=X2>W+U^R!jn2)eV`1}w;hk{82N=BMpi+y z**WX?kJV=p*39_HM0vqE$7+A}(@6E1Omp~TwQPHlT`KRTY~aY8r~|Gx&vl*ZXR(E~ zi$1WO>*>sFIWSSs)AP3bCD_HHS~DX)b+P+6hK_Ua_RwbNe;009x!fJFp_bikcpdfT8=AakfF~r$p498krFYnO!4;y{fpjrO*OE(C~>`l zXPW2Y1J5T`Bhl1Ls2mj5I!XWLlj&}w5q9WQEab0H`hEI$ax^iVN9SzuH|RN`qbjxa zNf%sUf{WEu9^omlnViu?(ZQd>EqXnj!<-fw4@{l-z0zlfORxbOy{e z#bUs!6DgEcR&M>PHQ?4OGiaIwq?QW*W6B{x9x4f2pX?h#0j9rKB7aDVLuETnCcI>J zW6w1`Z7<8`0i#^ItDtNBA^2CPXa>ZRdNf!F@EV-?yJ|Bun4c4^kXWpo+~!B`&3hT1 z5GLkyFec{J!*c1bw}~b;L#ahJI2@?aKR?sU1;u+hg}GZeRP443c)NREVjcmuF@SBr z70$9Jw0)eOvYy#L2L21sp~4klKYZ}A6YEZ zWXs{o*b;|HY3ul@*ez-_?{a1g&SJXDZtl3bX_s0cq)u!ox3h4*m_8MHaRbyZU~C3% zb}a0_&|uphlgadi_#-9bEU{qPHLbi1vHDV-{i2?+%sfXG3FDmyewby>U@i`mj`ll> zNhP{(Ta`%vFWTNQIMT3P*N#20?TIz9ZQJHV6Wg|J+qONiZBA@w_q^Ee+Ur}pYXA9u zrSf!DSErv!_i^I7?m@|I2hywJmu-IuVU}T$&040IV%JR$?)zuF|XrGL2|Z0N-=3Y9lePcbuT6N+EkO9q#KQHxif7P$%*T0#&x zh}&Q=ra&PZ6Ek7S9>w0-C9jH=cSoTLg>8I56~}M9$=RhGbZmZu+po3_Aa3DqI{Mo1p zJCCRW{UM1e^Y4S+CzBOA4>JE5Gp8I>oZ<^q?7|%wN>_C{bmaHZ;8yYQX)MY1z4>6| zyg^u9@Kd~79F0d#U;KK$3sfFdQ}0S30y<#c!a*Qj(m}va>`N?9{k~q#f>8M*)S}}( zU*b8jO$9In6Od#e8atpLX1&O1ptZu^YK6b*GHJR5oM?!=qJYSOqzQpgq`u*Rk(;)Q z0oF-Q&gSqom)gYc+q~W0qN)gBrLR~Kt<8i_2V;Y?%f<<$#})yVlK6274hf~CI0g$& zf`frTkun!_Wm@wA`H&9)d)R%7;0P+<83IKvg!DfYmQ`HNn;tz{b_xm{-K0j5v z_cJ{fl2`x@NOBmtLTC&pPafzyu)zB4jtA|KMDmy9Ka?Qlko=5e!GHmddobJvz{`!5o{W>e$zP$87j1vT;K%Z10x|&_ zPZUZMMHU6}mDue41i{o#1HwdM-FTDW>P`I(%uo!$a5PJUVm5##AL-a2$j|6&2s8u~ zUY3ccYEV>6!C{rMZc^#*gq^?x;cKSS>i`OdF*^<9y%+3}?e+k6(vWfm>N|buH(l^! z7UBcVH^xzDr5%`{m&yRPm&#WKf5<&oR%hKMU&cvojIe@?ri{|;X*d2r=LfiuVBa2n zkhVGxFi_o>CAgd~qQ|Q!knjSoFef9}3qQC&Gr7*Tnjqw! zn2HI_LRRVmJ&QAM+?ju%GKf|(BKTIa#ae+@leFpy!udEUxJ|97oetKSLSTvmME&7E zK3-3td+uO=uz_gdiNe3Ye$@fq;vJF6U4fhSVXEgH+>08i4Zd)$WOPS~$}jPX4jX zy$^mKBPyigCGxKa{P~6y-=g!h9o0hf#6@}4eS#j;=-_V>lm{S&GC81;&Ddc zAq_+*MZRrS2X@Qk!E@_9yWs4FtlT0e1@fu{_F}KDLY~HHBb`x;h?X{7yBsio=MQU{ zu$p=m3J^-Y`5J5lHhpAhK&S$!7_ZDpUIXWzmMbvR=ZEaw5_SrxN+1dNYZDprxgraW_p-zFA}d24?D$71+2yygQW<*R46MtaE)-) z)XZf9c;PU2|MJ3d+fpaZ2GwSDOoorE-7Z-cB>yo(f3%C8cx@m_1Y#J^x^7=Gs&dHu z4j7>i7-gjJ=7L6>U_=FM#6wOE`&;qiB*iZchn*B*AGY1IL~DI1jIO*VQuE8BxH$|6 zTLWwZu5$b5HsA}OzYaL$iHnty+TB{Qt-KE4@Rom;yDEWdEX?aprjA1~@dYA8_?}D9 zGJm(#r($|rCIY94zYUt);RkJMO6cV|Cp$hkE*i7(>;>G^UHx9>B+$(yz5#Nxy9nGn zR_IKj>P~ey#DX7Es;2vb&*5117o6^xtANu?a-oe(=y}2=+9tVrrd1B^pRC{F?#W`R z^XD)!HXw4>2nkDiP;tQFO6pq^(JlZR@BD(fZi`T(1E!V2jF9 zC#IC@mT8$bzK}^jZ8&Z2$S}KoYcCKaB_$#Uh5);8>O8aj`qmdg0Zl5PV>Je0w&q#} zikK1r>dDLAOIh7jJ5$o8vdkb*1Ci%K>w({hm08?_Ripv2(uSlwtvzh`|eIU0sd3uG9ScZT^_l z#_70TM}14lT+7IkXO$G%Ap+G2pd0zSCPte%Fm9yg_HcGY8UHsTpHlI`6~`J7`Ll;% zLb!ii?o^b~+p`tah3@o4)_T``!rgAK$mQw_mSg2zo`*f)W ztFH*=dA+HjB98n02sQ?s>Ri zZeA3;t3vara1K$Z4b$rSO)&#vErr1s^xPm?VI|sHg=*mw>G(EHKzo#O9cU~P5g$Zc z`xpPJKhosP{`(t|34FVtpV(Ci#3LeZ;AoG?dHgG|F%+hc7v6es^I|=mL=^hrm;Dy6 zZBEP7;an5?r**W`lY#HNBgW3sR!`qx7f?zf+|3Cd+@sf^L8pJu3MApOrC*~Z-`v#U z{uSw%$hfaTe@O2$r zV+nY>c2NuAKX!vq#!qL^L6{6yptyqp??5lWw(0NJh@QYbVu=M7vv)5XpWos98y&~q z)t1sM#Qet!le}GfpqviE;)|mbCY}5J4NaBKYz-=>^fVF;w26FI z-;ab8igVE_P|5xI9KkgK@zAp{ReUs}Dk+$6d`B8+7Gt;pUKCaOXxPXr&dEk3W|Vl6 zD)d;SP$eqWwl~Y?0tcTK zHCS^f8?@AdB^?>^FvHBrB=s(YIsr7vh;DjvO!bD&n5_y)NE}_poJb=QVB()go?9D; z)hV{;FX^YPZIDbCxX6?%?OG>Jm@b|kp1qeQ87i*V$5K=%1jU%qWh|a3*NI9|dPj9M za~rL8;?Be(*sXpVKHecDUR%v>G;}9RGASp;JgX4lFj{s6^x0nquCnHAuyChLycc=1 z_SB#jxS3K*M;Q!7VnD=hx`shIpW+_6H5~NTd4XBIokOjd9p7kf@>1(mAQZU!U`fj} zev!UKJAs|w?axIZeHUj-6e(Ben*7rIx~6cwNNRK=Nef6c%$K1-q{4~3MN`m5qe3Kr{dn1uLA}lF zxSUnW_}<~@Bbk%Jg3g$cYchzGm_8Ji^2096IP{ztRVLV+NjQP1AwC^0?I?FF_Es!` zXb#~glQB8T--ixNVu>;ZkY7_RSr1I=hzRDr=`}=u3K}N_hGJezHHoWk#vCpK`=-6Z z^syL}p6`eG?B^_Cb7O2t6Kye54z3YzOrh=7OAsxg=Q&j)s>#48pc^YNf z({#1L^di3dVJ2;3tN1i1pD<|J$*WaLd)I=vFl*kGZ>X!x zhVJD@7ahskf#;4NQ1<>>6JxQQ>}MTN3|$PumBGmA`F#CTQ{{4{siwAL>(g=l5m9R| zqP>D$+UWq35dNpfbg)==br@|R8&ca{ZD=u6I44!D#U7sUI20*qxTYh8%A;J9MXM{i z?vJXy42M?AMr}iJ8lhU_25{TYJ4U(UMVo-1CH1B{C1-J?1c|z+Ac>)SO%%E;Qxi%* zH`hBaRRoic3p}uCIN8mjZyk-u5=)L{YvV2?ykP|F2Xho~ zFMGVFfoa=GxaZDqRP=A2yDflqrVD99UJ{0? zHb#S^jLx%Kd^=AMEyABZp|d|J@x1NR3;kkptZ->A_FIDI(;upTafX`EDSGQteEX$w zwE3)lL+cW>ro~ikl3>KFp*`W-{!K{J#vSkNz<_?+wl`(EjssmLdYJ5vkB5@3taG^tnjsGC_j0c=Xu{q!cB;AlZDX%- z#bY@OlbfV*yc^6<^ZY!ClWDnYqr&>D6+ob z`jRQNW|(=E#P8FE*!zcuP3pFy^)Pp6U>PF05Q~5fpM}`4NBK^g39w6x*b9Dx+Ty$R zoO!Avhb~zvP9^dUs_2 zGpE#K&6dPV?l5Dh0hP5?_KUdYioTWi{fSfu52YPX@>@MGyM3sJXlYX?iOnb}2X|AF z0H5JPs|M|9h^KN^1%|^grWUZHD*=mcC@GIQ>$6vAY^Ja1a}baQ7CxSH0(3f2tAB5N z``Ybnl~Hicu7zHkLa7RFzq3l5aiPTfV>s}NbX*Dr#vnif|rc=R`Ku4x@?d*h_Z zfu+)aEwdtPo*r6Y^nO&{Qr8Z6ap?Pbo)b~_3#hFY1S580$sVMw@KrKDnq)Lvq?*v9 zJiR%cYZME z(7j^ZMes?JAhpr`>jz}i><^2mStX6ASr*Y{OdaBnQA{sb+2EtYMM560Y%XzISFW_- zQxik%lH)eZ%xb&WM>aF1%`rVHHgC-2O&#qPMt47GmWSxq-7xYe<2Ch$&YU1}Ur{hEA946~T}KCR=xn|`8)oX^RDb?5@t)@4F%pVla`YFoufZw^}&+nrk0Kx@c$80&Xh zmSOu0774+pzj|t8l$Yvz-1c`CJE9KCqp@A{*)_}{AOqgd$S|T0VlnR>f#@;wzJ45h z__l@cq}B-OIven`fmt&~knJC5B)ppGU{Z&anzX#^}sgT0`PqyRGQ=& zA(5IOBe?>V>nh8yiKOp5g6Fy`CGAU;rRr%R@IoZ_ZN$UE)eAu|v`~_3P4_lJS1FAx zRMo;%XW|9XR^$05JYUy9*NBZGv;B{ueMF?Sc!n4pu}wOd(U7Q2Y*ue^BX%&%Ya}u7 zxyzzp@Z1OfV}tIlK%=losnVP`Ii%hiwx25AzX3k}3%@#iq0J+pm(EJ)T?ewEH9Y`dUq)l(tJ#l=O>nP zE1j#Zc@IX_(pAH6Gg!pvunqm^S*Fj$H4jb0#*juOV&`@he?U*IRD@5QEm>9H+VjQt zt-1cF8KBzS*+IGE-9Rb1bl({CPLlbg-|$u`JyBh+$xdHAvtH9=sl$wE?N7UHr(zW% z2l|-?LL#hNPnqAGk?6E(R|a#C@AfrRKfxe%My9sYo9GBnak{w{zFF9JLA7Mxyz$tY zzcC?0JEH2ka2F#9n`agPo|oAw>nwR6SnGizm`2677${#N^(PA+8*{O`-R`V@t&q<>wjZF#0OOaHoZf1=m-6{3l2|IAB3B4FaXr=g>t$=v~h>NZNj{)6&W6hh! z;=w@giG+k*8!)xTA!hS$>9MF3g9$wXEqjYoyeqs@t@SKCiLY#1sq+HwBnfj>L$_Jk z1CfN3Rua52Q8Dc1edX83mGl~?vix^}Ae_*f>stQMc5^6_7`;Q!lfS~gl~B4x=`-6F zAidr*fqqJh-taNy!ohDTvC;IKkRaQ546p8>W`B$3`hH2tjkP9Jv3}}Ym5{KX@ZSn>v~2KdWGkMyC6hhFA~Y}B~f1%i=5=gX0~8n8dW)m$+?y@;D9;~ z+IfJ;Hzd2Ka9n?yc`DqlrJ3M7bsqAeok;(%9XZ3zaeh3%w742rOInZnj=%g@=M(#59*lzD{D$g|d%R{Fi}}9_{Zm=Z2nf6G#5Dhh ztWW*^24HT!Am?d&Sr5>K3w7uB6hP)RAANZ5KFa$C&f5Gnc83VYtvb3&8?VORn+{44|H2zzBA|V|C9cZv^?fh&~Xa6XMGfVA z9hz(2TRD+@LI)7i>M)@sKbWs&=rdUq5p2=h{2oUklfnC*aS;f9k(s9L18X$&IVTy; zrBn9smI{Vcb8UP2g!RhPZ?*MMO`#+owWy;Af%Nz+R|pa~#Y9wv{>*5>GxIY^pl^_F zaYU&yxfh;nPd_VD(zLX?;LH1w=AIHv*l#GK+VFf2io1n68z7iTH zy&I4$rnHg{fs#IOPL`)*n)g;M8?O}T3{8|pn}QU^D&+c>LU&F>+gqbehT6ptzXSK2 z;v0ABO1FexT`cp%OQL_O!k=5K^}8rdGFw!GR?uLw&TLu3->IEat6|=?Sg*rFK1rK> z0AH!~V}>{Kdhdd)gHAKsq1&7K~sL?P3+(Lzo_slI>@&{ zBzpzuJW%DIuG0bbfPZ&L@H-(WoY*SC-uc`vs>=Nx0HJjU5y?IaW}SkQmi8J2Xgd^z&qos}YUeBKW?s2m``AY&!?R3GS!>hGK*z}e$o zEtlEnj=E!>njmAI0-c6CK2$3+&n@xOC4b2D)7^bVk9uHUlFGq=wP9YO=B+T|32~3p zv$NsfJs2K2tB5N8{1W^nvzhM2Z71`hba)RK1^Bnb%lP&Wfk(%t99lsh)Vd~5w8cs} zxDL1)LLT&NAQbdmCqd-BAy4$VREX%Kg7jY?7j&P?Mjs z5vZidtKZI<_3MhZ#RjdiVcfkx58uRsV6n0`GjdS(Hdq$Jo;j%Ye-e5Er^JwNV+M`w z)NyFll~F^*mqGuQdeArff2E!s4I-&D_CLg)-b7akB{8nivPjZl3=}#oLG(URZ80U0 zS|fN&d%4accbjU@xAL5Ed+#hNabh39&LB*Ra{Wkrwi@3>E4l1|rM}5O`ef-IyqwP; zw1&R|8<93R!7GJtr!|0^kT$UKDoO3LW8jm)Tj$Tdqj+#XR1{uki+=b6o|05KV2M8s zpzrj_eBR!ff2Bd?Xntd>2~L9hyi*z=Gg>m2+2|x!wb|z;ka_0|`p7>nvw4X7RGnj8 z=Dx`M7F<#iZM6RVyN#Kv6#)T4aMqT_)4}=upxtjleuVh#mZa_R>;EnOY`y-s_Oi*S!0(JJ=>Dq?M@KkdGadVXUrC0%)=TM0MWdo#dC~&Z6SdyOUC_3q zTvDdEH$@vBo5xwRPq|HZ8HHT7#Gwp(oAN~z`;T~)CkIrpjG=bquyNb*G^Da_#_t$a zO6l6Qd*X_8N=^x~bjo?sZPJy>crPcAELoI^=Z#C+i)!jGM_Cr(?KrOX5=j}Z_6AKj zZl;5lwnt+{mbNEpl1rNl2~y>oD+v+HTN^);l-v8vsHzoXKw6Q z==Zlelo4#poQo1%Ra)eE5Ce(GZ`xFM*q0ZO-95_FZnr_;@+hBVp2_$j2f=guXzR|X zkX<_lF5;Hoq^+EIT->;rC8b7^a2_`@4Cdrb`sZ4tOo-=4{NhOoVS0Y<`9a9-Wb-`D zPaB?ba5Njf^l{ZK|HO#0?chWnl^=GS7vgRD53~|(spHlXZMAb%5^atb7ZPp8AGDHf zrTx~DZB~a>lB*4_9g2_2PC^gXQmX|FE~UC`iQnZroXx4K?LXw{s_ehbDz{#8=4;nq zYUZDpncL-N{76x@nX<>}CjV5qBQPP>n)F$Ksgm@~v;YxCAxsCF1rw*LgpP%oO%e4q z$KapzMYW%Y8!d+m-TS~u2ctxpCk-VRD4vsxT4RB}EhHW7U*Cjz>-fbJzo2DaVBq7z z8|+x(ClnW$=q4&qSdwD(aZwVhBvcgnW|A=^vS>i#2*4oZO_s9j_&b5sECkujkeFhCa67!$%bubG05_vK|&YR+6-=Y8s+n??Wiyt@?e~GhvjH4 zm0Ie#Qy_R)d&rE|Aj#zgB6uGHr+6;tx3hMiypKMAjsV3p)I#32w)g{l@Fk2G;`#m> zxa|H9D5`Fa30y(>xUEwqD5_f=uHf1Q31AMWY5b%DkcX!|HNU(34R=`><*0Gf9Ui%& z7+CIEJHPSnN^)=*LoEkQe!cNmcu#RWTKi7~=3);{uOEMoHtHr2nD6At!Iq}DDK@=pkssupQq`0Y8s^g7#Z6cq za?2oPc*F~HBHd?**qEQ>N2MGPCfh=Sk2AZxT9EHef(=kQZOvq;ZpSVMz3 z)<=ySG4Gs(!JzZv)bAk$B9=yj5p<%6Vg?f0uf!#U?l zNiyOCG^PQG6gQZ)LSrqD;V&p?tB@5Dcw=jf;CuT=g8pa_;vAdU<|mGARCH#XH&B zi>$6ccKSq1lQ8qcY2n}7oiDG)aKOjGH0L^WAzQOFoBJ8O`IG@k<|6I2N0 zB^QwgSTsdQa3r#S=(wZVFSwYI_upG9!-ACStJ5;b%m%S1sp{66`rl$n^ei+f$8DEz z+ss3uWtX`dNGy^jG|9!3=87C!7o`2=OlofZ`QnGxT)AonX#S(G6)y&TlO{jcd@l-t z9TawEJ$g8Qzb$y94d(pzZn7+k-~k!e-p^_Cp{W1Dop{1iWF^w0)B~O-}tSn zsudaRjq0_vG};!PYKkZOyV*K4{D=2~Il&XOJAe7^j+ArCKhOqGZws3S1=jNt4$8P(l63-AN zP;z6~FJakq`1I(8gq6N_5acvu3jd1pv-oxGla`~&7$N>_jT6w93o0k)eqHDQH+z%j zW#v?fa=gHECHOXojh?V^v_2eWw)~I4{Zjej)Xdg4=KoaUU!r+w&h;Fd$X2V?9hS&v z`Kpp(jq0PBs;K#oJoh}OY%~+~vh6xIdK2}gUzRTCMt>ZqFS8$Itvc2(oE|HuBt|$l zL6?k}`BCjhA1jl3SoGTP-1ctty*s{0h(iUUwGpSOc5o~Zd=lQ-kGP@er4#x(9-;2+ zm_r2zaV6P_J1^w7Gi?@I%n^^J(kPpgiuqEnt$?QG@-`bX=eK~IxRUX{4%e1;8jU!* z@av~Jg>ce&NnJw)vn!0K)kZ2Z{k{b~Ug#InvR2v8UeL!KgRNNSJd(!{dS{=rx$s-&88;&whGy zH`jwnWyk20zXTqs;NJj>Dy3SE`pOdbZCRLAaDAO?VY5L6xi=BxO1ZCAW+I|d)6S|o zc`ejpTnO;h&_JnyxX6$?gB6M0D8e71MhGE>7r&4OX0vlR7s8#Sk0d;-4=!sP;4SQm zx9N_giTS2Jh$GWYU7)(%ztQ}1m61l|oC%=eh-gnfC-_1%owVQtqJlR#vW7|K4%VQ2 z*N#!u?fwGel}XYhD9A)w7@;tkHXiJc6E)6pi+F0(09e31bDXfXO6zW)_*-S~S zrq!o~?0WKM7-8pN=+D=?f@)S5m116TBpDPTShoWMHdL zs3Cjv&x6h%x6OfG1-75aoGA*(K#$_fK$}bMa5P&fN%DVMe-kr9wkE@^(sZ?ChhfHD z{H07@wtG@J8r?gq5$XCT1ozgP;da+{&v%b$YY7{}|2PIVf@|bz$2QUL=~rMcLfu@T z`;)7xypYK1-KczA8yD7yaRgDmjlwmEOGE-Rzb&R^pl+<2a-eS0_~J*kT9{2N6di0%eiwDkSRW$Fd2ufNNwv3gB9-ER{b6uAE;>uy&Iirhup{i zO`}{u=Xu98R&%dAm#+rK?ut^Oz=~)eP{UEHAu#DTmMwPFyLE0#vrT){>lguYfqxS} znxZ7rZGKxs3Mf|4fZ<<_Li57d+=H9%VDCINT9YGc5(>ki!^z1zxD1c-ymH$x1g2Rj+N--_(N6&{SRTcBTzvdXCcXTyIGEYJxo07(f_qV)vRra8;}~)=W{z82 ztw*wW5<>`eF1~tt3nUW9;n9#DLUi(n#*QI^7cWIBj450_26kN9wS-j@*?*EO!QsD1 zHr4Jg$);1i5K;jh!t4bVOdDEtnJtKUM&_3v${R)=4<3$_PY+^#FKq+UIS^WE^~d!` zP(hVsUBzrNjt~e?SD`+NjW>z9@d#Y$s^xa7LaCYFBU5gBV8F!ATQS8n(J+41D4p@o zE!2^6Cve}Qx3EpZ$u{N8FT|R>&!e6qXMtNMfe4B1bq>`z;9qh!V8@pq z-Kf0~!ylnkG>_rt^wq^dV!w`sB}e(Q3rt8atj}~h=o9~JGM2(<*EuS(?ZFZl(cfwG z52QVH0U#}9%U`55BMYrMNyrkat`BGhU!>PSU#{?OucX7_#Ep*6{xnlPnqy2%+ZL5U zDk-1l#LJfGNP-d}dQC+Yn5v(Y?Vk^GSIh*hJ$dH%!zZ1aBygfLku3%cQF3Z4YLN`= zYy_-PVY+cNK`Ygn2`H;IDt2_1Zz@wA!%$QDww5`2VxY*c|fjDx(bY+XEH)8wD%9A~!UhZoopS|05aP4vST<=MY8!A73B;W~=eEw3ikJ)dTvf!qoXB<(?P`#2XG~<%9HBuVC*Oh77ai2qy5P zQo=%1^yJYY?MryWNT@yJc!t%>=hUk`)ZRROKn}DU=^h{TG?-vQ#}?KgP?xWS$OWkj zCwhf$;QN?2$E#cq2Y{&;5pxVGqB5P*M%cy916apAclhIa(;5gT|55@OJm#P90wxxE zdN?)pJuV;K@+?n@+%Hlt=WU)}5+3)7Fh67PR||OD8^xv_tXA#sXg);rHtQM?Z`6f^ zzadH0&mh(*D+#AOAB;s;f4A|OWe?FC(vcu73VpZ*NtZVC>`G4&UIx2@zR-NhW)OIklbPVz~Og zaIRhca2S(B6pERMGxGPXSE7(k0)WAh!H)9;vgxzkw@m>Gj7xMav;iimw`8u6JG3Wi zI~RRba#70WVFu3Po5&LbN0?SIuvCM=eufTC|I2|b4e0)}diVl7O>F2wrNS<1-Ea5; zyJdp31-c~#8nlTelwrsYYtn}Za#zc^oJN?bP#1PnRhL9&;wkQLnT?Hl2M5gp-wdy2 zIM6T;>EIc?$$JxwW-$`_?4KpQ2I2kq6)3O!)f-}r1cmJa`dC4UkZsMdgd{ONr>`4F z;WPNip0w2xQJ&``6Gc6~@7T)si9nFHhO{87%!w=|roE=%ETH5b_-Vi%DCXU!U2v&3 z3b{Juy2uaq{W0ll0Z~Z` zJL#JwX@de@R$C_L|*J>09uAw+;~EbGYo7S=nuoJO}-NB^KZv!cV@Pf$1IiJH{w>_G=5_ z){{dL2Dp6-KrI_;hi?!c=1M5~owqY+MGAf=Mc{+ zX!np}IYTS%)1@)bm?A0>d4rhKVG~I**DYL#zW76U6h=fq^5*`&B z*Ua-sWRvE_2r{P?(tZFd4d1`H)L;dlkv$NsS?HV?vc4I1;Ko<*DA=0smR*)e$m3uZ z_Ms`5Z!H3L+aB|xgI?J$2#QcN=SPDSaf95cyw)a1m8+J{=)8&5zF zG#A#y9M)LtJvM*$vixVe18Z5|vC!7q#d+)NFVXyvw4D& z_Q{(FGF{n60(?z?H+z-F2ZwBiyHESC*^E8vzq1(`^G$~Tn9X3_HJAuGhfe^wL|tdL z>d7WqjvaeF$tIJZ%a@tX?*9#Fo}L{M)MI9W6h?{G8&p4+JP(}@!dN!{0W^pKxNU^Z zPdgcg0GFr}#{oYOqj+pb$0P8C9b1^l5S|PdY0J6PRu)MgP}p!zOkw3R!a)Q%ehtS)BK8)dxy}ga`QyB?dIA}&+*UOX8IId$5NX(X8)D;msRGd%53fTkbC?dn>D`zvTb%hC5 zJT~q4*I}m@U_K*jqR6u9YrU}%@@_F~k+tC!)+zm^uFz0%RXB|i%L@?)uv)*#^Ar+( z|1k(ycEaa>BAn;lzl5Vf_vipFTvKKBKNF5FfN)_FFXAxEZaA3$cm*lv6K9wja+<|qNY4_%v-oJLKgOfKSga@ z_0GxLCi#8jQV`Ar9#u;C3k^N$FX0Rt|0Ntno>lz!)GK#%-yEQ;l*C`6JYgN;dp;~F z8G|Cl>a&#mS)k+)1uhX%o)<4II@vxf#4%^jYKOlIK0X5v;=O&S;J>Y1*dEQ=_^khV ze8QEzf&ShDiqpuC`66P*i2>-`U>1|(HlPTv!b|JHwKD~C$iBSxEJHZ++=I_^g}$W3 zlAI)U%*%XkS9tS%Do@FMzFl8mr>6qNgziYdXA*LPk%$)LUgZ(Rz+TLRUR#7^5lZrA zyM3@w+bQmZGvV|4N$CycPM{>U-E#;Oe!MJvywGj?faLVAFsVw|&IHx;Zvo}%JU7B_ zt8%1MN`xO8N9ls*W<6y$%S=So-D7z1$$)Bis6ehovs5p(s%8fbHx`M9v< z>w<JN26wIRh4uXF13ppURl}rJPXC!0yJF-5kNq*$;p3E0ngTcK6xlo(8t?HT?SS zRmE!wU#=|eAn$gGvy|YT>_FkCCtw-Cv-f4 ztb?pA-->mB&=9bOT596(dDAf3C6&mwY)0& z=0gVt#JVG?-QWXZ@(aPu;D}#qvi_qTTG<%WeB_c}^T>kto!t$8QwscKY{a(`MVLF>^7Z_|)p5%>?shH9^WP>35fE6_&PzK5nzsmwJ||O?XS;jNhtmv})r? zzANSG;W!s2!uTNpj+4wo#icdvpyfJ0nd7#YDnUKErV zU2Zsuqyli1?<^2wG@2Hr2R4=U98lP*)pxMV`0{m*RmtN(+GYH@mZ$uOqj&X+D9v{N zwNN2OdQEHDiqrSKf<z$8|Rj4q@mxBkKqD%5B7&hwFhEuB7>$`fV@~vn(UYr^*)zy|XiB!=3 zkrri~9ogmmBFFZ%%U6iUKCfpixXV620wexilz8wcz$j!8@vS!BpbyQVi3eCPQ4hk* zU&K8)>pmwL%aFKh=rTN0Tk8mi?MpIuA>2G)^L02|bEZym|28Z|wrL$N9CM~b(|pAn z${+sIdq|=oDcFdP7ltZf#TJSFC~7Y5%ycD;DV*LH!_9TTIKj~H4?LD@FT+uyqVQr8g4;W zbj?;r!%E^tSt&Y8x3`ChsJ`GCdtnDR+2-X(~XDEQ_Gn=4RzovGeNos)o)r zMn-+C%kz5n)Gaj2zc_S|Oxig~if;Qo?)| z(90Em^4rJu$g~Lx1UHOfrQi$qKHmh}q3XUxpb)AtLEiS5{T1~$BTQF{&_vGNHdbO% zNubdI@F3E5yvUqN$6Z$Sc4_J+9B@F|Un8TM3TgPhPCge*P@*mAp6|Z>>+B|VlN|G( z1OgMMBX4x5&A{tsneVa^+B4O_r|$&!B++|m<4kzHtgaQ6f6YtLg8V#UwWwB)qTuK2_BLCp)(lF-rq-zAioq zI9(&TPwkm_{&<8Y_M6u1($MWv<2dK)vBcKu?b1+^wq}rMMAUr+uk*tQzy@a;+#LXF z)(kxm;f}?S(cP6*d)Ex&iu}6>>k)Z#{b_Zi+B_N)R|t6R#L`WT0T+SN(CUdX5rVm2 z@Eeeb4(MhfIe5x8p2Pc1E~RTay>ndNY5ZGoFuFiLSo}zq7uzj^c!q5|O+KjoC<9jN z2@_(V(dt5B`cK=xTIDpozpd?p|3X^sWw|nW+uT>VYc8ysmR9PxhMcYp5!7+2-ZnRN z|Ha)#GWz3gGyV{w3pql+jywi5%Dp2v5qe?f46iGBdp&UBejqXiBS%46bI-FEeV$i! zvKjop$*^ylEiXF18|9b=|63UrBYDUF|HxW$h9?*utRA~q1{j%LHeA)Y- zt#b7D|Jo|Y3HUdya;5Qq@RkSGbWQYAmOBH9f&AJNUtOjbLOMIhHGJ}!BbtxCW>-M* zeD|x5miq@d%1>ZYjM>h=Jnx;60lrT+cA6HQ+Mg0n>NCfNwj|=h6M*^x5PWYOmtVM8 zx&^~@eTY=r9~o|WfLOcB`T}4f|Gh424$x)e{pbG4T4)ov`swTl&E49seQyU(JGk&Eqq>3Zs>_5RSK$0N`Ca86P2pHKR{^Ne@{df8@`5*3g^LO5q zNBn=%m&bq57xTQ7zTdI*H$W_%T_@~MEIqqVPNcSY%lP;wWSuK|J0O@I2rVk{jVerg z{D2PpcWXTNDb6x9`q@g#7Zb^%n~W?&L9RAEokMK=fPAHgPSpY0{b)O?=XitIX1Ykc zv9E>VVYI|f$Hk#4Bh1{sUQL#`q{FUWZ89VOaWoZ8qQlONp%Vwf&|?|KKfpU;glxU7>71ES!g~ zUYMk}h|I~4^4Z1dO{)%>0<01a{;k>T3z|QjobsXAK4`2Hv7$3^D?k9=k>$GJQYJ3c z3BFxt-{z|K^o>a%ZCdpfZ!*0-7XD06M)#{`umO7sfc={rl@<3^HDYx*Er}U&#B$Bq zxfOl{BrQVRB(>zN{VPk6=lLf~`7lvvFM9d-#nr7_0C6g0TvY)9D!336PR)J5)~6tY z&$;-_q_aPRj%ovj!z=(BEZz7#?Gf{W1q1JQE#q1r^{s)y^L$&Z7mvOLKRk2Kr8`Ea;1iL zfa_>u)?pN0ADH{DT^_O9_*w7sc%qv*Y}w&P+%bAzVjfx!{*dV%ZsZ zG5K1pz*!pPAk=0Q{Nc6?KAb&OggxYVoJX>89emYDQ_aH$QlT-4dNJd^I`#4%q-dTo z_Bo>|M@Qy!TqASSseo`|cJ1aQs2jJEwXq~=lPPzdG-h!{R{&Aem`T`Jf2R+35T{tb zvPSB9kw;p`x5_+uacRMJ4>RTVN$^EJ`g`%=do~b^VD>ll@2b%}w*aC(=zH=*>+aec z-eU{Ph5aH2z#Qz(@*bVPE_Yjc|0Ij}xMF+nn%e!)=~{Td45xn&xQPzF)k1V)wFt$G zT5F-=E&?qXn0Bki;uTMK=5`J^QU;pyTDAL(J3l=tUkQ71-?T`3a!~!Tc;&CVa}EB> z@Ll-IcM|fHR~{e@y60FsG2cf0sO}8kAJJ-D#~*W@&)kA@{1FqLn(qfUhM!oGanypq zr4JcfvHH+12FNufZ9vqo8m}p174Fk;#5-xm+drgHN>n;a&56kF8rO!2uop8qf+e)| zToXAD#tASahJwCY!d`j@NtT`PY_LRx-9(87kYLCUAQcq*h<2O!U=D>v)7b4!XT(%OI_W6XQtP3 zp2ViXXt+d`o#jqOiSnO&MgSeNoxnBwW$j zr^^GX8xw7sxAd0Tszid}N5wf+Rp~_KN{#Gs)?wMmvmtyOvF)=VOnkT?q7(vH>Z~2t z-Uqm#=iG7#|JwdmaFFjL;@1R`M=f06Vg$sIil1bJ@~LH_a|96#c7IHzk7f`7zv<7Y zDEbm_rYK!nyxN5ZKI+TPRfK*|PMvVxZTey$Jon}S`stQ`G98vXk@y#%ZaKkIf`_OD;34-GpC&la;-cWzu3 zE`cVk@Z+i=AChVyFYTKVIeb+V?)a~hvl~2BO%3cxr-$>5eFR7*(3b;48CML8`ef&D(rx)N;;rRY`yqbmpB~| zuQ7M#WW6`v(%VdC0URG|u;2F;5hh-8bz?XB7avs|jPa-ez*=F!sS8g9SSx#eAYX3s zaG!l4ILX7EPcZS}4|K6_zbSQFp#=Z~ycLuG@K*fI+`6j?Hv?7}2BBg;V}U;Fu~rfW zZwsTb#!OI&(uT@k*HNWXOD53`ULhfGMv2NVaou3k=#nR{!TIMRz0k-mCqVx91jt$u z7M|&RtW0V~Yu3=0=3URL1%dODuwM!KKD7uhfYPvxQ7ZgSTD$5Jp~AUWt&s$SJgRql z^hit+QPwqI#CLrYd$1KeK=aoy%zt#a(vLbR*lVn4tt@MFAG4RA&Pr^US4 zpL*|39Eb?8YfPoW^OjZh!^awKiq(KH>=0C$#UN@FDY488Y)!JS2pLODPjjH7W zG96+6k?HupIU_3fvY6Dx*SJ5+#GwzcKrUL^h`=U#DQog}Mf2RDM z?`UA`2EFMDFa5TDc$T?e{_;C93x;UhV_5sM7{R|EYp9IuA2tYJ@ZY2mPOsH<;!k%~ zQ2SR5Nw0MY#xrGv5Qor(`>}iN--*Rrlk@7C)6ajYuV$=v{P+6mBKO_e8v#4V}Eg;?T>^Sa;4gIk~ z(#ig}R>d8q{J(b=K=Ry5?{&qGXCuQ4p)LE(7j0BHh!{X+Uf zvrS~>+J8nn;$}X6M?0RjQQ>`mJ0ZOP-U(?&w;Nq{i1C6!03xO1$g${ICIW`Qx5L;( zpt7GvgXVGLXASsoxmlrqlbdbmx1O7oC4Xh;Hf@EmF~8c<+tu~iADnk)SpJC=eDm`B z+v@0`aDt%o32U)Fy1j9Z+b90s&9Eb^=YiMQ2@huzLs#WXbaC^WGSr}hSHTZ#MjD1- zZA-yUhl0pP4P-BWAi@PdF*9h2-nxoI9*ih#g}((ube~jB|3oiE`qRNL8f?=MhwA(! zt^I}Ln7k#dz}|(ZFi%-T_iv=Zau|suW}}QkI;sz|5eR-a*URUZ54(()3uLFy7Rl&L z{qf4L&;Mw%Vk9}5u4evevqrh+iz58BS|wDNMK*T^Sgl$GtX8=eg@^uGt+I#A8Fs#Z zg?+>S#wNFq{vEniK`@MxrZ6VF#Uptg6RDq!My0phdz>8>+DH7;>YoCtp)k|RR`!@o zEtFRIF>3Qf#U30W)sfu!LfG&0RFoe@rqXs5eW|W!dneVYMRi|(>TzC6M+u$I9tsiW zx1Erqh$wsPf+i?9@EH=lLr&QVsA=>$-|j=;H7mb&^=H0N!aPUG_f87dwA=kk3Sv-O z%;!JU;z@Au2R@LHjsqk-2b>lBIf)HoRRmA}>j-oaf!%OlCkV)~o>#Um(V>gNFp|5c zWxM1@J6n{1&>p&)23Go-EVJc@D8z0zn~DF(mvxFDjy~CC1VVzYPPXy;tsA-LRlaxW z_vokO=kFPBP=}jHTIbj^>$4C;A2LhCx>i#{Eg?w%8!7+n@>nX)F-k7l zXM>X;Sm(2>vr}aX-Me&E`mZUoY&C5XRaOl;F`0?r*)n9=%do0Ucn?4%4t=dOeZ@nX zX@^U}5huHf_nkerY~gU1XMuaz3Q;SRPy<22x&452gXg({Cb=H^d9RPK+wh)IV@|n7 z8f>INdM4F6MP1>OtglU?a`lWE5{zp0Z7p!E;asyT5Id64R&aI|Vel1mW%=wE8b$Zt z6IrYf9J|T{C|wI_rW83u1iRE~O+S_aLsR2{&BiJ^VWyz4CHtTAu?;`B-r^}k;+E8s zj`GZxp3G|V*@L~W{L+=kI83z`h*rF;8T}FjWXP^7M76lwN%Yu0#*<^f^O;~xth;dv zKrx)~<;EU0E-d&WPBWD3XPT7LKJj_6&M)pO@(s7VY#hBg@}% z5AJgeO$W=71pTTIvX1A&(zeq{$fFw}_EtdiLu*Lo4sS$>Jz=x$(05d?*3#U~XJ zpL}r?&ywPF9}8tBMgZUeQG5Z&WL+J`7fH3Z+7)YcPbTaHB^ge4CA(?$>2fzK?dfu8 z9o2C4fJ-x0e6?b2&wwAeI8-7l-SCjy@Sye-bqeP!{pq`cwJZ19InJMc@43Q2)_`oB zpZT@^ZVaBI>s(6ndj&i55P1qxzd~`<_r)cqF;Lz9hVh$Q!(uyy^kx-KosD`id^`dA zt`zIPH=Bx@8_V~{CagVV`P-skvdwEX?>widqvp(Y$YrAhH2cJw$yzN8dGoY>ID5KO zT`fLjGxyg-DTtpkx%=5ZK@wUi3fQ`NoZhrd-aPYB03jo~od{|-R!y6flN%gkn8v^`2s_>7Phidre9AUoW7=$zE=a@)pyaPco*A&?wYlzF=yb@LY&LRF92OymA zyHkbvl~Rvs(l8n$sqam_am)nymB<%Q(r81moMgJ@f@J#SA<9Pw8k_n2&GXy(FD#EM zqCqKVp29V>l1&Mx&!~&df$CUwtZPy7SY*Kq?9N=B$Ix6K28u(_6K&z&yhlzf_Uyqv z?6a#@!=nQ{sUu(HElL6jn#{{u=w7bA_m29q$25;X?K@tv5iC&0hq1S}VzkHobkAIB zyD6G{ZkbC|QTeq#UAFNQzjo1U&}_nHxH=nf&7!@p$G;;>3SD(>rM%Zux!+Q`UBkDR zt!8AA-dy~-t#E9xDejGmv!3=mT4#`ZaV1;i!${+K5HfmDTQ_KSjV}E1ip-#-w@SM( zy1{GT?0pX{3rps0!2DN#diJpCDT3}EMCGAR{{9snLMcM1aFz60T8r;en>PjEr_`hwKijkzM}yU&DIJa)~m=GiqS%&(t|8_)o;& zzEoF_Soj0j*KRIYgVzWn`*n#}d3D1w+w#(m}Op{PY?XbxR5|s)`p65QtW;s_V`3!-*c$Ge| z57taa&D%r}g*1(jDx{szFaLhXk%<&JNS&E9jc5jTBqQHQ{Lp?HR^mAi3;6_sRhTJ0 z)<~R1_?8J1d6E#ybDUKKSeHrUmQlq&uA#t4f^8)3yj3o4qnk>??rnqFPSQ=tIDfVJ zb1hv7e4QeJZUI}lB=g{BxIAJll=)yeSIHtw)=4o~9~3N8CC~J|U$jvwFv+B)l8OEI z9OjAhm~F=u1lO-_&PyK%jGDBG0{5j;WI64UKV0a)Cu(0r{s^;GQ{JtZ2ZB#UR)$bb zg(@b-fslFbaAcBh_$HjtTR9)BRRUM{16jAgDM=o>ND)})bTt=wLwQh2Aw2#z;!&pO z&wQH>JD2*mnGZlVFgW-_njFY+!25&QUeTV&6$nCEeBh1x>u2ACYQGA)hcfb<IT zV0zQoLyc6*LNDSuybJ`V)(iHLY|PD#4HKo-=soISSrRXJDvwfzUb7=p(z8{o{IY+- zcEAP4b}*SAPI;0Ey`q$>)__>`gU6M8A%-5KEYJ&3^hc~FY?dBxP!#7NQG|oDpW&() z0Xs8;J7qAS%ilIhE>>~%DO;U3R#E6rhBYZfPA)<>JB*WzVNO={&tl$fKx9d-4E&)q z!X=(2MJ+U^t;@@ijjcQ}fJSIeSGGL|ahf-yBCvv011+wVpheZJLpDD#kQ5$Qh`;g` z3|KDkEa$pBKP*+nXAo>fG9+V+O7P&k1nNuSY7`ox8?_Qt0gOI$3|I1|03v-NZO6$SwF_VzZKUp~Wzb(bn+_HI~s>pdFUOv%ld414%NqNvY zcCGm^U96YkaMo3n`FJJct3H0T-wF%_468*lv>D*+0v3xZq{ZqE(U%fH_1fYqp!_8E zbu99=(0XUfmOm1lZ>90vkM5%QX4i!PO)S#j`Qf`j=wnI$Y}#XL>|(tfb(txF%&W<& z81Y+w7B*d;?Kjrfyp?Pre7W5kph{$&dDQ8MoI)SN=t3W}09G(%lDGMZl?328Z!MCi z)Gt%OJ)d2z%0YL?fFA9j;C74)SOes~vj!|7>I%Dr-YdCVnZQ;t{?f;nJLt*%)~+ny z=dzOb%_Fip=;p8f^|xPuC4k#~mod!mXU_Y}$@QWFa8NH2RTM^7D%-jPG0Iiwt z7g!C0Db8#86G@kx*{+b2(s+E|%#Q{}s~1_qgkMCWes%lf#?SZ#vh(2D zm7smLB~KOFg0`Z7`F@@C#{s3rqqqwJ2fHCBjTab4tE67}#Qv9FAr&vCU3Op(jWjZ9 z1xhXP^rx>P=UAa6U&Hi)7bIFtBZ@#t{mEpG0Dvv-OGz zTjiW~Gl(b|m99o&v(!zB(oq7Fp$YKGX%aSrx+?4TO-A;K$*-}ni-YN|c&aDh5ZNg| zcd|GfF)qj-%Pek}9?dB(lH+DpWu#Ol_hms7+tAM*Ldq&`b6VC6o0F^@EyoW7v&K_Y zD9=@K>atd`%bQLLmXfezPRJ9(@M58SG*{;!W0NhON-q+Ob)2agMKJk5%NkF;z-V04^%m+DE~zM^=X0my}A?a>Hctd}bg;M;qxl6WiIa zu@ZZK_1xIccdC{eU__RLaq&f$#Wj?BHrC7`e$P5{&*w6c44@zAaMz=>Nf*rZ+hd{}K+kU31cTcik#mTW6 zs@?0-Uc=6CqA6Mue$+5F5uOVMMNk;F541Rdc1=_Y%$@Fh`wR_v(2XU`B=jj>bz2a^ z6U>X)hY5qR+w_ryU=I?y;_vd?BMMUD%VKIAtr{ zVE>uc7#0gbW)hLfGbHr+qweR`h2{EU7v-h5w;S5={IkX=desk7sS5Wf1I|$Az4?X= z!0;JY&kR{@XVn`I$arQ7j)V>+&{Vv7KMl<~I{LMS`llZuLKEDxiH+>R zk1LB!ywfcTBmmjwQG%;q6drT?2`xk!>bTRFFGCLlCc=e%9s^0?Cq%~dkei9G(zz_` zkM?U+fn6WVQFI5j_5N4Q>IHLdy>yZ*+#2@zJ0jukm$fo^5r@~z&VrH| z; zyDRR;)~@2+HX`2bV6(KCEfrt2nyrKg4)fy168QSQNc%p(U3v7;R%cp#H76_JhSOH7 zoawpCEH@Xmve4W%Z_X!Y<5-UO*1uv&k-Z_S=1LkRKo$ygaNix$T3QmA{t%e5+)a;> zfa(O6#{B|{NmYp)aBaZZ^riNp9Y`uw+7@!IL6QVVfl!pN%`8hk4#-^3_d;IGf@BKA zGXCn^r)pbD08BC*U#{?$;{L7J!3>@DY_6EUXF`Tz)aCQpP>x)F8kcm^Re(C*82RCr zE9LrE?$Yj)Sv{V?euOW9!pkn>)5)LeepHzyk(1z}@oB@UstorKbD<jxc6lqYYq+mUN96-y&ToCC%q|j&SQUUW`&8d26M|rg&-qljD4puJso;5JaxoeK@pDI;qx2U633*$=O_ zd1A)mmfF_F+@=|h`u zMz_FZFfb=vO8Zv^jf2R0lj{9t|737EY@QY`o@abrY?=}h7Ij4o*rakYHg!AU)p@-U z3O_OgxFo>Usb0MiMM;0MJJM(fDSiFw{GPZbA2LQpbwx3g>U=s(8VxCn>iwYllAbs@ zP^nH9^;ta$UKoEeDJr<6@pv9=niIk(EIZgN78koWm)kE;$vp{NLjF{aJ$j>nTSN8x z*65E0F2ivjY8D<$P^o$9*h$0If5loQz%sGU>BtK7eJn_jLtNz#`@CYF^R^tsS6|c2 zG`%VC`XKDyvJ7HPjKw_7CuWk*O>b2DM{o$$=kz`wNj-rA zphRp*5b8V9bOdV8FSyc1H?*S_1ZjokJ#T9YwM!FM?1QlwDEGG6oFhP^mrT zP|m+s2{!WCEWvPYRsJqBEg7+?V8X7B*%?uAP4>QXdtmA4FredEposF zZH$>ZeJ)e-QiDS<#li-=-9eT!v!diSBz#bm{FI7gK-7C zk!r^`xsc>sje*hniKUzc({Y3{V`ncoHZGY27@+r+Z97*AD?ZBB#Dbuf!zwwNCIE zWARJ$nJcU>)`)<2R`CA!J3FIB6&5LY4+~wdbGd!NfSp8dd!GbSN!$7H=TA)@+>9xl zIh0w4uTIQDfg$RkAfcj)Q_0QV2}LtpTdX znJomL=0g+EYH(urLB|v@|5On)`)QSDgaoJ_7AoipduA{dTil%R;=;{eijz4-Rmy~+ zf(}z!OUi=b;`e8;UJh(dH4~8~1szk`oB{A;z28snsSojhl1DUHQ(>s^O$9enjS*$I ztB7OI*)H@=brk|(cI@dM*VG4L{{3m}9S8iQ;4=AzZ{G&PCzPx5P?pKMIm;3{aKU-I z-G#VU`a>gGJ3Rce3JpYUowQMsPn`%^TuWfV-j0+sjb#+DYbOjCc^%k*WlIW7h21Pd z7VK>q11Y2f)zEx!zKe=dm~t1`EgL$Vd8MA-NJJ{E`WYxRq->cq@Xfvc0yYB8X|SIr zqj^HPUO7FGzS{F=htXR-6YR>QQODqcY1lyQK57Fb7~?`An1D1V+7r*?){ii^GVP1{ zxY*V}6uVg3G`dXv7Iz&eAwk*WtmvU>9lHfoX4j`g6IK8?p4|m<@r1iZbtnk9zKrOh zBCQ^)@wXUmh2Ot}3LM|jp|=5xVvuz{oP}~|3+WCG$F;Ga>*@p_K7d?bhwbB+z|@Fi zCU3;yIBsCsO=P$7mspDnEEVkzbqg8wa&Hw(axO1XwYp?21&FTRE)cxYE~@ZqRv{=Y*0xl&2r3t zSMEzf1jF@I6q#N-i11>0VH|VPcvELnyp#wX7|iFM)yIO_BJHlsb4Xc?a*oHojR1Dv zhCZi?%J4}n$7+`TuJNjzC4zW-q_ z&(f&rC%RRxn8epOwU4gqYdi0`=ki&|+OhtzAt*}lD^5pIG`>sgvZv*_071vnlG54u z)S^5Nx6Q56i}gTbKR&jx*uELrB8Fuy3|d8Je_t%$>dXMXnZeJO0nKD+K$-3GL}O_Q zrg&UX3A1^Z6zF?4v0Ka<*=T!5stNHl$V#|I-TYCgN*O(gbz@}}H_G(Rpc zP6Yf<1aSWu6Yr;Teq1=QJmVa13^+s^pq0A!QCGCjO+-YvhaJ{UUIL(ulrbQ^67cQf z?oWYXsW3Ztd%Dp*;g_)Ct_)Bd`fOytNHYy3$w*=F55h(X50idSr}+eK*9V~Zc)dMd zTYT%;IWyJ#kRNv7ZzDU^)>l_~J!3l#pWN|DUBa5h4Xfjax1rMX{SM{{3#`uqfQ7)S zk7C9F^|igWX;E+BD?ois5Ec^K8`XZi2el1SnsE=90Thp+pi+vSAr%NCzHYCgxwJHG zpmk;b4%lx%#X3s6%3z8JrJVv(h!;VBHk=rm^-8=bY7=+WSZnK1;7F8e`G{zR;v^O6 z;NCbIyC;#6zkFsS?%n36jX?elGTKa#gw&$}=Tel1}Lb z7KIUPk-hy6uBqE8z|g1Q;^vaYMkkc7`z~{R66gtg7hm7spZA&KQP%$i^(%0hK?w~J zk@Bz(t5>_DCmiS-!u>kA(WW>u;-!`l}K8#o@=wexOC&pLvfI-FhSSigErZhdHO zId+|RUmS&K^~|C<0qgBh*l_{ZbC83!SA0s+Yw`kJ(H7JZftEVK+sajU!sKDtcri|CGhA`I z5&UI_mbvGB@3~McvV#3+&xBu_o}dR;^rDs3YArAUc3U9wGYL&@eG9)``}_j0nzlN)(OCfEDPlF` zju9g!)Yie#PbRcaCONr4qP0CXGxu9EeGw7-YJ150_(BZ^{fP?@rX6x6GV9BM&2!9ePpIF}F$MXc?8b9Nu&Yt`9x)_F`F%A_B*@57em zK$p|bz}X>7-6C10g#A^(i&Pb?iDegzt@M=W6kyX!B?)bn7B*(U*?BG2l^0sT*)4gj zN@D{cfYO;lLb(kf?9;PIf-)fa(SE)yHe5I(m>et3`+&c!W8b?{zq*1yS|FaUEC;p< z@~O^OBGAgyG^*dn_Ns3*vZmkI{^>I$QKD~W(jJ=0&iq!O%gNb%nv>j6fNEBtwU1Qw7Iv4n>w_cb##Oh8cSw3oajMZU zvtieP3OW<)0>_4qXGMnvCtZXx%NU2eb85nQY39=>yWrcmoY?gE87jdb&4GF~KWT*T z?Xw;R?=iI8>VI$WI-I`Sn7A}l0h{Z4#*gA!gz~^xlP7fDPlh>uPG6Xi025FIni~m} zE%e2lJ8=;L*}a}P)y0MsC=x*1a47FII5-2xS*fbsN2t*vC>da?RhA`k0#RMr7!=Tu zG=fcla^9;qut*pbO+_78rhH(T71cOkxBPv<)FMHFfo(KJrQ|b|Gx;8;UyWy2bnaE~ z#ttRw{NUK$jsY1Q7jVf`aA~BkOw;Kxk^}~a%C3=(Qq=S|xeh~fNz(Fmq2{SfzYI)G zy5sT=2{LS+{c&0!yc>BmS}s0zfToZsVyR$psvi9OP`Q3UDMV4N436K-P@u5j@~0eE zAUJZ%a0zc%oTvIqUpQP^GG-f=9jonP-@Isys5HxVN5A}JaMDD1>=;7v9+5^x5>lSP5q^j2^-!jgn>pZtI}API0%Ufq~_#5N6gry{>v8z@t!^LlCg z&zQVSdi4RbJ_;bB%soCcDJrRQRK(2E(@#=9%L~h)RyLZ4x79kA`Zygg7T<&raR9FZ z`>PU&M!L|VcMvcnpOC_va=BnRnIU)$`8u3FYskY5v1(mKlPsx<#J8LTUqx_9ksg7P z6cyM49+wh)NjTprc$0|I$}Z3t$nzk*k}cjBiwianJCivpqa-*3y}61_3uH5F`@7n# zX#oly26lQi6r6QssgBTQQzEB2rmI+P5lw8aKv1KK7kQC@n649Z}~Xvd0C$p{iRrR2~h z$}szu(@m^HcB+&!*du!NvN=B6_0XMz@j)su4z#DiOzMmn2{ckPEi@DEg^33}(=6J+ ziDzKpxmeH*zAA&nS(VIanuPQ9<`P%JOK)oHS{olKNya2tJHsN4CVL>>M<@= zsT`X1G|*+o!4&C|^|F_Y1xaK2XJd1AnyfD+ReW0bWa{!kBjGt!OztPBzn>VqO0QDF zMC4sc;WN(}D42h3(yamaw7k#abK5@A|JhIj;(4$8Rjz_rQeD{cynDnvWh=}wl$T`^ zzF|SKoPO1})8qviLF~n3!@ztom(8i_Yq?`d&M(0N$EDUza2?k3R%Y;`nC?pdLI#@w zsJ~dkZ}`g0R)|Qgv+j@oG{mWYgsEdl1=2#6fakF23LKZNwS3eO%NuI5{wx&L~G2w#V z`i%W&cHVpR7_cu8vLAxo`*x18jOz_rKl{t|ji`5(aS(pV1M zX|M<7Stc0DyafOK_81foSAhRpr)M+2AYgME@%;eyCBE<5*5;O&e{$c+r$-*~YgF6V zkgpU-T_`LApkplpl3R;VU;)s2Q$(pmJ`zLDxsmUnn@ElWJcumsl!qz)R8B}K`h#G4 zkX(9_I?1|vOdxdR#zR`RDk`kuj&&tRV4&e;o475yLs`P)E^-V3ZutHA7$T@rcJ}1oV_6~qmUU(k{7)=P>Ia$8JggD&Psuk^ z YDY?D*py7)*q$mgL+Em!FHk=s*BDNp333*QkS`eu^wI$ps3WTUp!th#_C9@dc< z1=X+RFN$bu{-UW4###SWH?eTMOX>w&K6>B;(q;$GC_5JvC2P@`-W zgGimvnM7c%^A$w!1BBN(cOI8}IH)|G`p$}+>fQ28~V>eN*MRiz^<~^(s7-c85SVe1lzxA3pW9gGV$>ljW4^;`X2ftP6Y1h zimNbzj)8F*(ZAP6(x-7|q9nJaHc8^gHWYS`>K)*N!?s9Vn=d%$Hp!mAgcBDDy`|Cz zQ6brVp;bDY>dPZ>caU%TK5m54%)8FL4(7izLx=g`EZFsWTRa)K`BKM>-NUQinX7y7 zqdFs8rL}?l)RE;!is7!hWs6$pDJ;lX~BJX z5^>Mby;lY2nmH&F!2wK6rUaS#7b7wPK?#t9s9!*B&1+t`@jmMZYDovUEG+EQ;jZ!u zU$Rvx8t*Tj$TwZnD?2TTtjHBBy}zd8F|7bT+at7pU|3FZqv9W=3e)c8mimPWv<~KNCV+!o9ja zWWGZOrClc~(=LxzdLX<#aQy5)+{&?|C(Dg)()GIRu=*m4a@a&D77JpQ7`D%6Xk2;w zxnjms>AZEeNjECl=5`b3+z%l8(qXpqKvP`xjIEAv_w&LXrBy4h{8D!6d#A(1y;v}s zA8sq0eW07au`b4O$(Q}IbYz3mQw4f?b)(cji6qtiD$_zrIxy1C-i-IO)QDr_yIeN4 zcy~NCm9!4+^{_HZj1E`&o+;~<#83odW!0;Uj|{u!aW2g0{jhQ>(#5aEE+fk)nSWsC z(A8j~t2|rYakT`}A|VPZLxx?ayA_l+(gLrBkl^CIi{ahz-u4;x&cr>|fXN;Sqxu%p zH{+qserD{nMSQl(obJcBaP=yyZ`hJNKvw4Daag}_D;nC%&$OeOviv!(6QxiIo=SU@o|(+F(^S5}sdXS24yF)?c`IEId^ zKG=c#W+ztrB9v68@|wft)_@-St<+T#c}+Kdn#(YndKu}SDqYwu?3JrCwtOsydt%vU z##pR2hD#DD_}T7EyCo57>f{^F22_%Eah_m@cMNDC;Bd5I*(~N%vHEGurC8>k3ze@d zPu9z3Ed#za<`?8!PlL-RH0HOm%!acz*k~iy7+AW;iesQ9E)5B5%hGQ{7bWJ1T5>L`Ce$21&>mD<4o61B%c3mIi$mS{v=`+B)zBlIjbUYpd^Zq-*J2zq zT_Wx(0pMJ$IEUN*1hiG>%XYsAd~?gm*RLS^cA!>PK|9%`5i}#5en?dt*vDCtUlS}f zR19a#Tn1(}`pg_>5(4qd3TB+Pzta0>{nnxsR_eAmgQ+~BC4_2oFM%d#y|(fss9Bh0Pg3I1v#EEoUv=&xofr!=v|l_K)f7xbY^K9`&S38F8juRo&2S0A$6W>>qJ?ls_!`i ze=jnZ?7}fC#@)IMfcCmw>Sv0A&;F8m7R6LyVR(W^yagLI}VI8`z6uSk9w z=U(jXUy*#?yWSN5$%}x1Y70D;ciHI6&o(&< z0p$WO+lw0k&cVCUG0Efibq1V+M+goY$sfjfs28RKT9{%Bgy{#LqPSFmkuvTRKC*J7 zs;#0jp}%2q>I&4eTgon0xm0r$1GR|)o0|o}cp`!nnC86Cr)3R)QCDH@o@1XkzkL^T z9=}c#ZXxnJwvoX0_t?hxKVlo$-o8rh3y}XA+b{~n#KYJtN?17teWvS>G>*Zmo``jh zTE;njjv>-}yHF`cZGDs7nw>X^eXjz9HZB#8#N~9j13P4@J+!MV-+-T4W)u~`Xs0(? zT2}n{qU^ZAkNhi`r85k!gW{;NI)dMmfBa8~T{n9%5K1-+5`SSwZFc!6Dpl<<2v~=cqbCRfiNgg@lS@ z&{hAgP{;p^LLE#1AX;X3e^;n2{#Bvo`t~=48kkyM^WHEM`8}YMAW`=JG4_tZwe{<^ zZ!nY0I5W0qY&)5;ZQHhO+nKR#+qP}n95?S;d)>3|J^S3cUwYN}GHO(f>b?K|?P;w& zlL%tThMZstdCjUHFY68l&w$@0LTRe3+K%<;bDol`lHEL zBrEt10#^>*V(6pwmk*eCKszdlLw^=>z9s)MP&DY+f|LzE##$mR}(3jA|rdudSx3d zAir}hvQA-Fot5YMPwIUl;OWd%rj|tC)=yoI;7gD&GJ~X;@%Ns5`@^$l!qMRG$+b^!uVB1X8{0?UFxKDRw>G_E zF0H$`m*n=6#x#2%%70;F>wTQ1-PO2B#onAI#h5IAoi9>LiiFWFTTO~G1Z3wkr=&c( zj^8dh(1~o=hvDx1YVL%H9-5HAKl&ZZPj|eId-d8>#^8ElIE*=N}F zauOm1T6d6vhXVbKx?E9`!jq?Yjx%ll=#4INoL5KN@hrzB4>417wS#Ed=%~F)tVFN{25&mhM^4`M{?%-)=o1aNh z^f7P2fk`L}o)r?>AOmG{~VVDmy2l6TQ4dt*pX3T)Ffh=Y3i_ zzupe4k}a5kGlncJfV4xIQ~Zt9No76|reUx=JjqA1*7nO`Of$@5SgXBKmw6(EwmP2V zevZ2)(hm%-I14yY(Nh1>taxZJ^q8*I%HUwIUb#$0N5$>B)pd1yrHlGjkn^8eTJoRO zogX&av3YZ%zhPG78)hFiIYGZaZEf#FZ`rKQ$}9xGKkNGUoWo!Di2ODm@=y^lg=ucn z?QpRO#cMcSVk*Af@?u!(12v##TH^ob(uXKqSs6r&7tmOKnT$0&+Yu}> z6BsHJNLi-DhBk8a@F!tIVrDjU;h;16`-WC*X@`RU9OB#$t=Xb21>Vy0 zNgWBxlh3l}IU&pT5Q^?Gl7C^&)ruIpy?VHsVE|6H<*%>n17_9$;z=tsru+U)uTS6f zx}sgyDiv4&8+Hx4m%*ac>pIDcV(Mm?Q|Olru8wJ{)^}pYFVU)mvoEm@AD?x~$(>)h z9?oKm#HHWzdel#gzmrSB5XvZGicn00KAipm6@Xds8dzg&Su7mI}s_)n@#0Zloptm0OQSN|BTj zo%ERcm{C?@GgGC$okuj+3++Hl*|~1wCs~zZFZRe3Lz)`M6zXFr3XKl>ACTMV5L)`$ z26m`}R~qEh2~@}CLw%ZtH!)sshLy@bWBi#yY-K$3$Jo^T9D9-^%oe__Ay>Uglvv6% zrUZJ6EI!~~GnBU&6ZG;WXe90lKYl8c|1G9!{kW zTdm4rp;}3kUN>XV$g~Fcac=Z$VUi|vOugwjIqnuQQ-OAWHm}t(@+X@@{Oxp$dttzD z<+@6BYDuMO4$P3UFo{r*Xr5BS6Duw(UfFR)M{A}e+BGbyZuO{qbR6Cfzr@J=OtXEG z%}_o%=|ZWRr@!(>dyp-eWPpe|BbG8{~cYG z{tI1m{)4W%-=)9*XLMDtC=m#U#{Dj&$t4 z9tpDpc0~cRH@0;U&(8$d7qH(Qecq_3YR8aur~g6M)Ul7(i9it?bmAxM+Gr6-)I>do zb0Ie5TW#XnPDisX3`dCkEifY!njmQr>&Sm}XywD%>*sff{J4+i?dxLz&ILUDGh8;T zaL7AEK(A>7ujdlp)pN+_|KO-N{cj>PIbz4i^9?i#CtBa}a3^j6+9n3-y)L5dfTj2k z!aNIw>JW04?dw2?E^H`_t~%kEr0!KQAeaCo*s$ULHPpu-b6N zpey(Iy$STcWk#2^C!y;^l|)S(2xVq}_`fX1`XZwR zKXiae;0Qq?kyRkwUTQPniDq55dbTCXpu`U932@rGd>dY)w+#V%$e$d%$o=1Eineo zlE(N@NH1CIqpNm+s;iL-dk9Hj0j&P8es<}#rCR~(|NPXVBNN)IUEFmtzWD(3TAZ73s2k3r-xwir7YNf`Sah8vvuOljjVhK^ zb`+;kYwgs~s-tlmj46voiFeV%3D66>k91PgJ8ToDlVdTpv@C;VeoD8Lp!W1s)v$T{ zm3X_P(j;<&LmX|-qVasSZkXiz znJp<0v$P_d*`N%z3c2U(^cQwR-{J~kNlfA5Q=R3;d_%i@L(8dbx&J_sUHyzCVrZ*c zxl+f3sC&iTXGNWNh2*Qc5lNcJM@7qF>AiW#yK?QQ%Ffj4Xhi+IM|~G5j>|wI4V^P| zeVyy3sg{e%Oo0%eBs2;VaW~IvA~x!VU4Hpi5Q5a=DO!^l)g_9H7`^FrB}D)0tbQ@V z``Pj&(?D8nb!yJqU$*et;!fgkdE_%;^mk~u6f2N2WZRi1!3OxZE@$rMV`=?;+5|f6 zM{PurKIafo`z_vFkR55z^!`DBOxAvl@-Ci%`yT{OW6?dF&IB7XHy0#i;_d<7Iy^&i ze&^lp>`E*r13T5Xtm%TgOi>BEZK3Tg@#iZ*`Rn@k;LSJv=Egn zg=!Y5EF>Gy?-TNvitGM z+hMDNkc^jYWhk4@W2=sMQzoMnkA9MK2D5s&o-3@)wkN;o(t6Au%1jevpvIseLW&KBRl!Jv9|Eg z0eG<9OfPnBA1~yXB7J+F_!R3kRlY@uDX7!AYrYpFsBehXJ$wvURW$jVB=cyXwv@2oKf@}A6xTztfjFIG=8hG(byK~Xhwz1o@d=atKodKxPIg5=e)uOEo7FWza#5s3Cp!0H- z={R`v(Q<+UMX%F1Nj*>Lk|JcNO8Mq%K3hDmOKbdV5T0}A&=q;={z zdFGy4Zj(rk>;W0p*i>Gj;-ni9;LI6_{uhWgJ`V9vO!dKo%5+v4!%NkOz^=LwrIGV9 z;44j6r`Sm{I1b@Z2dMsW_tLNxbj3w{^&!{k*=c2_rEL&b6JaIca%R$kM&!6pg_z(& zmG0%j(AdRRBA^E_vE!aBd{aQJAs`2lDgJ-#!AF@z>}#8S{p`WV1|jiSo#|z1u>O7= zqV}V|bWIg?Vr=$gQRSz6L-GTMN2X<$`e~o_=O!okn9B@-fkl81Yo8eWeP^$uuO|7 zrb;8OEfo`EHn?Vq{f2`-x(Kkzd|Y;W>iy^crqqMMG!oRk1dkF%lc+G9o#DDll|_#b z{=25iM5t~sONR<;Fgqi=ogqYYTSZ3#YcQuOX()-wf$O3_H?r}jTY;(0Z?HH6X0TX7 zz@i+5dG>BS)ubHtk8L?Xr)uN+3O6L`;oi*QR{PSLv8qB zX_OLG9&=2k!IUHK_E$PiE9Q?A+SeQ~%o7 zMFcEpHP}ZlRuc{AG#(p+osr+}FyPU~-n`|iY}UJl+6KFHsBjpOL7?c+A+@%qeV6Z_ z=tv>-c2pz`Bycz}?{@%tWjN~GTKg0KTKfeAEJ~4oe;eg6|Ap}@aHmmZ-D5kjm6$-pDFtByff%YJb1MzcC7hruBaSf2NLqz+b&$ z0EL_xRx!2fM_VG%4QM_tq64yG+om|T7QT%^KmuwIlt{6wuLh|spRKylK=?61`O=2b zH~3$cy^n|^dVh0&{99?269b|0(dWOc*F*nUubuwOdfnN-^r^C-f#T9?WK^Ie($vU4 z#Es`)@!y#Lf45f4ivE9Es~vuvX}3Eyk1xs4CQ~WFW~kET_ig`G9J`BNTr=bJb4-;q z3pDL-fkTtck^Uj`J!OQX+AK{utZRp}Fk)BdPl<8>HW3H;!dNbJlpPprP(lhSJyR3e zmufVD{Fm7za7YDi8D6yfy?g8~z%R#1MH<3C+6r?-uU%znAYPnW0V+W|x?+(y_uw=f z`hYq8cilsCxfb@f?LkB$^`pl%^m%RLq9{3SPEVClM0MzRr3&F8_xE+R0u z?M~eX&A5@1)@1joKV1^l9kE?uL*uc$;+raMwayAFYU@msWFkXH7K4lL$2^Wjy?f-v zA~}rD8Gmj**M+y>6&DM>kfe~Kqk$Hz_0_Oj$2v>XbW}IK*L)Jj#*%!Z%_YYxTlO;3 zx&>UW3Nt`pj&81nZx;1JD1A$*_V7n2huYTKYv=dhkCEmE1Y@vHzQh+rGE!Vd0?VyH zS(345W`c#$IrriKRB@*TP4=jz>P&G5R&_H!Q!RCJv;=n-L*X3z0reZ8R0>wEh?$6~ z?{&Lko5Z$~_jaL%Dk?-~%e6m1M%OA&x?to8WRuap2ggr~r1`&Q-`&bQ=1e07?VH-fm`3~^w0WlF-sTY1p2vDpUC zclK^=8FFm+uyeS&x$3Te>-}K?(RaC>t8HCZf;_;}XCMIse~FfE$$I@6B#k>qAR%tG z6_2$+zTBRn+^wnS`A?qTKLch(zt*pxBLGj|XQaRXk0TPCa7chj&NfTF5q}Su;q)8j zu~NlGaGKcPe_QTGdLc}|cm;lTC4aQ8Fop4oMYiSNWjc7m0ltwTMS7dCqaPQZQd z5(dc>y{Kr@x0Sl(i(?DK9w6sFzhIK`(9?=BPFJ%a`50lP3F_e*v#b zij&8w*2@W|f-%sOG9SLqXY8vQ8(`MxZh+bn>Jqf={;(Vk*!A{N#Rl{T#9;tje(}4V zc(p%*e(wnO!=@XgIrB%N2%7sVz#ssW5Pi*$^$FQQ%$K$v-?DLED0{O`d=h8{gUiB^U?#_rk5zdi1>i3ic#A(A(Lb$>o`g^JOJ&%ibQ z$KOsGZ{2F)=UezgIa-4f+2xfJ`#B5YnvjG`bO)xk>O;2_p5?c~cY^@()OhINtmumL z!?9#jVupm%Q+)aM=73hd!TTT59ot|I?A1iJj{GUvuHkyWMP;3w?G1bXcx55WQ2 z932^yN>L8%uxL-0c0#cVeN<7;GslA}{*T`mR3`?}EA09>u5h)`q$`7a6x+^jZDKv5 zWT8q-4ijKm^e0Nc>{5=p3=f3YgTOHPaTfGOiY9sun?%Ph+3i7^@p&R}L(<@B5TOU+ zB$yt&q%1d<(q%Md!A(T(3GBWX4Y}-@YilRP6HBR%4hZEvc#u&?wpo+unMWqYlT4z^ zjTp;trPNg>y~l@X5Qk$I=9Nfh%kwVSB~p8a5z^4voq%1_)I{;VEQIZAP}Sl&nsNXH zhT}MUXS{IEree&GZ=w`aoJw+xzrG_AgRg)b-m{EX0FA%#XZF(@$d3_EaRAq2+-7BB z%-rB7c^wAgTap)5u#v=t*-jhrt;vLUh&JikDxA`+BdT4}__J_@0;$X>r*=`ZS`DR@HkWAJES?ypz`RabkEbhVw5jaR09D7AZ`o$GaIaRNn@7oL0$kBLqq;rGqEx zBBZHmU3)tI0dSq(6kn)<0I?Vrs!~9{o0rs^kB#gCAmD4}1knDoxd5@ zr1=Z>$RvUl&19ApIzqhDiU5%Jh}=5)Jo3x{u7y7sR&fMLIi5yNF<^J{`JM@DLmUBO z5+@G4A~%c=GuHHo*xlw67`|{LEXEwd?JP#2_GTj7c%oia>Fm>@-XkpT*7NykM=6!w zUX|sV9iKJ)?YuH>hNQ($NDKGQ?+nxPqbEk1fhF6}D3cF!G7NT!a!kJ#<T*VOp1`oVL+>~_96%ZGb5i)4-0|{ z&znd)%k=mh?CvMS9OzC%}14@|Ko5eR(Hmz{Y11k!r38n_zFE1?O>W;m7V8J z#xTksW#70Esg%0`=`Z#`1cZzH^l@1@jc{ykt!P^WS~WehXty%Ax!^&Zh)GyUSt~J2 z?<(T#$MN9_<1H2OvM%MP@K(y?6o5O~;5RvU^(hrFy%tHg=s&69j@pFQ(h7{92>MES z^|{np@|plOtsJ-$S8q^yJ!;JiKoIMW6a)>?Rj;iLZ2NCkwvdj$Zlh#UHS zW`z20~m`IHZ#* zA$nHH>)y-QGlUVjz+Rs{cD+*R_B;#p$s}Tlt1uB&ZJ(9uYVT?LN$PdptO@+R&-x%$ zk(hf)KaCJ2CjF3)>=nHbaJVlM&r&5M+G6{hWf_FMzD)rGLXMko^l(i+{?o0uYn{c3 zP~&sGW{D;jK$>l@P;+uu6*9Zhz2dAA%4gKRDp6c6H+7Wjl`l$b3G=u{OS!k7EQx10Z~8lX^o`$Z zz(+mu4&%^FPF}_UEuR9syITN^*w~nWY);6mneG}y!s$nwU@OEL?tFq26ot}+x|c{rbWPs zJYCDd57}Y>+Otpa(N_p3)M2XfQ;U4drffF}L`60weJrl;aJBC=-LLtLKjSv8B1KI- z&IZ?0G%IoJ4Ns_{G~y#DJ;7*Q`X3+9Oty_Fj~W>+r;ztfMpe{GZrf9Shy@qD@zynM4t?^zzk3LM^0J|) zKLWA>3qQN{{X)e!MxWLk@WNk)4Tt zvu*m!iWdXx6WT%e5(Xj01f_8563v*IAPM`b@Z^o_^qCM&Km`;$F^muOtE9A5k4zL$ zF#oceI{hU-zem)b8NPlg{mIFC5ZblCVwDLR9bZai8PNe;zS3se}0Q>=!k~UV#27Go9H!e!$nc2 zhyBpMm?|$w6J7K*H{C1Hi1@9O{uzNz2BS_@A{@!cg`>iPqAZ8;O#V^~Nvs_7d*gCXrR(-ZzK%~LuP3q68F+~eZYn0)%$PK`m~5v6zxx|U%8 z&jIv1Ez5pXPQS(%n{HRY@>Z`OJHUbT`&XmE3l8uIgUEHOl@M6}L`TC=3=lqz(d;V^ z?M4=0mK0hVFSJIot%dx+#*KukL}aO4A^Kru(fxzw-s=x-C5YXDW%dRL8ZFuljVnzC zDLzPht)&pF@uA%B*c?-(mUJo(aoTzHw$?rjqyWxVmGYvFDlM}X3M!D}cSfueQ;nu6wK zL5R$aNU+3rf3(2HYLJdNS$Tmj-2B#igVU(?Rw#m8mwRf+zcj9n zNKwphwr!a(QEqj7wmSgMq`8w(G!mPDME4llE#Sqg8neGXqhHdmmvZpOdqog-(v>w) za@()Bk6h5Ku*}`~Ws^7pjT3Xw0jSl`=Gp}$XkbAR(XnirRzhlyJ?qG~P7E*8#Z&gr zwnm7?ms?WjaZ+0kU6tAPBb6OFw!xvUGFcN$Mcpx!9>fJi3RX!{Igc=7$-K9PTI`hK znVzrZ?crGPG6#;L`>#I1qehNaM(YCj%=g_L7qzZ{`K`ZcfNMh2OUE;g+2@&M^mB%N zvC%+JDTeq+k!zirKD934Ph@lXemOpI+rJDF zqqI22^(OuXOnGRsF(K@LFyxtfT~PJO&2WZPkY;Bq^=o~0pBo(0YEVQt2>yUrrcVwF zH&oz{tu$vjffu2O!Adfw3Cu1CeJ2`&hvXa^?Z-U}xAp#%uv9LnzfEL-hho;z1JV9? z)6|8J(lQr7bL?@T$@%$Wv%@Zci((3c6B2YgJMBw^`t^c{Pt)^sUqfaPTog*sbWd1@ z1SIdfFCN*C-8Uq!Hv^D}4ktK47UeHX0b_^`4YuT!VS+4DKsMjdB%L{Bk|E(V{5CvS zUr@fX+Rn1uzn1sjB4p-L@_Yp#K8&@;190Kryx?^~to){D%l$yPT-lFa;2+FqvVTwJ z;`&H&s=d_`#2bIs0qk#57B(zG_EN{PsC@36o* z`kVZRn6&TIh2k#$&v9t?w5qXDNkp_qfX;*AzQc*Q6WzsgueQ%maA}-FZ6Bbgw?@NQ zO+T;dNOSu`fhl0we)&NmS_qQ_zTvQS0!Tpd{U2!#8$;%72p%+;KeYOJ4g5IpX-)US zMn9=|Q(IP=>Pk_^D#x%i;yk!ZLkddYN_-ZsjW{5Ha3zNnE}#a65?9DLEAF%BThNYU z{ot#4KiR%+ZZDs&zu;ZLsLp9=8*a6}ulBFIvj8_2A@09(OH9%*$`@Wjx zSjXqtL`Q)9i=%(OnkUU`kmb`H;^O=wruxGZ_3L?9Wg5?`uzppAgA?HHN3u0_pOqO3 z>`UV{v3H9Wl;6R%VF*1u1J)1X2WWPAVrUhQe*R9O8unYebZ-K89)r^9{rqcGhC=&l zlLvX&g(eY0$D{>#UtsUe1oJL&=z`8vi}2t(yYTr8$osTmqI(8xN~x{^1V|AuzP%Az zw%cim7!N#DNB61Kp6a*_3s?`1T1R?!Z`k^*n-}9liR^f+`^fw09PNLTu{u72diY1} z?JCELc9UswrZFxf-7I^#o&+!SWT+cb0YvdnEIMJ4H`v#sO43i zl{Hs;_u--jS@c3~FS2XKX9P#(1OUxOG0N*xeB8#5h`8 zgi_*?9oK0U)&>I`tKE#32{EKQ(*LN6XLDrZXAs0aSKG<05=vk|j%9)3288W=Ju&ZRR3r=@vE;6e|3dUz`yRe zr^f8nRk4~I%8VpZtW5{urq(zNeMXcaM&KRQ zDxpU&*~gaU1{+LEgiy;-#alFAy$#uk<2Zz75_Jey0=%XCppy-}gdwF~De7M7o@&?b zr<6MpR_`j0K!z2mY19N;7c{zbWN3$_sp!~e-_`s&5SpPz3FN=F9#rh@hqEB}32pkeaz=#_rIfi$O-YMw zCmJJ81$>A8<@bma=pY~Lu03ZYZt^9U5v`u~iVZyKER=6|$<%%ySZ{W>%3T>RDSJD` zT9ik|QfVYxzbBh2fAmH9E{Hv2+w-u{kn|dner$!gz%mujV8VY`3R_WQw>F?uFTK$1 zjF^?IA%DNa?}Nn>#?wB4JUMXL}6PYVT} zzD--!{dFethHYI_KT@(nwK;J~s(CvsnBND&lC$h-W(1RVL8R6yW%?5f_m-w$Rt<#{ z^0R)|OFt;M88Fj2juls(NOA6X{1ECvn0_+EPVsOo<=Z;hBi2IAIzWuV`k`je zuWGera6N1o1LbL)1WMBG3`Fq>!<$CRK8W59CTo+U5PtIx`XEan2scs(87h3espT_3 zUJ0tReUx9x)|1$!fbZeW`_Mtu%8mh|!USmbBdV`7`qh{2s+^cY%W}J?a@g|uo?#z9 z5HJEaIfF*u=mCeSUQmUoN%22Xa}I^Y>@0|oW);~OsKWWz)WOP9Jctg9y&JG<=tCAw zxHx$lFi}W9H>(6Ocu=6122-Y&pPbzzmQt_@@tQj<;=k4rSsLOcnVK3 zMLt9h+`~ajs-R~)%oz^|NAe}y+VB>us~Q#$6!nZflMjHZQgBe!oO#c)hRs8jugV53 zCEhFK+>)(ecyIeAmVM~G!ze6Lt4Y!#V8#pNRlXj@3WDFyaAXO?P<~K4iD^JqPz=u{ znK;T`9ky)#07se;k-e3k%4Z9YS&>&G?HXC7eHu9Z1EUP$HYKxtL`^Wrb zS!I-Ao#!DLb>3U_!Jh4-B~NfOtg={KNIS4trJq--uEdcW&$pjUr8UaXPH6uvktkL) z{+JPIwrNKJIP{P%>`tm0N3mM%qD;2INhUCafQcyo_W3%JBXY$@Sw>0NV=wDTo6u)Mo$SX;yU;Z|6rIsE~pR2NwpSoRSy0SMGKMYgm z!%wFj?}5_-r`!{O9hY}3sT{dN4dzn^gqVA=2`JX5DdR)&46}Msn35u! zqdKgox>!N7rp3HYJya|yM*X2YO2s?)Pq~qJW|eiBjUf$<@Rl0TVG4u_&5fDSw${n_t zc5xX#^ZEMtN=~OM(TkcY;^%Mh_E8kD;T@b+u;aRl;hp7%(C$ss*TNr#T*0b6!zsKJ($%WbCBPJbrNqZxZEX#5~RD*jSgAvwAwb0KY9H*`acr zuRp3r-&rPEOUo2!rEHP}7Tu-?dk51Ar8f2Vw??txa>H&szs8u8(Jb|`XY$mm9Onc9 zJQ~Qk8C=9$II$Yl;4g~`feFM0M+md5cM)P0!QXG{0(C5vu$^!Undb%EBF)lv)5{h7 zMv5%G>UCPvt~2&aln=i(75^znS`hmEmX(UCPVd$gtR&L1;G;mM!c(PPS0;G6|nui4^PuPX!yJbCb2IkR=s2?p|*u@jiotfi+N&Z)bW#t;{1Y=C&l^}SB@5+uYZQbF2!Vl_iwk}=~O^rJ& zJ1YZ);6M$ja{_`D#UUn<;NGJ_R-+^-9C-=Ms}eL3%hgr*_y@qyB4Y5qMw(q?u8USI z+ecmZGLBE&(ba0xLre=PLL?He=*ZVNS}kb})DcQWiQPm8*>q>`WNns7$Hhs5*VM(R zUf|`(uMI1{>G0IhZr>^WsHzGrDy<`d@i`RLDtQj}ZV72Xx90BLOR~{8qh_n=?!;?? zr(=0VELgZvc=@Q|p})i#%>1A~8hJ9gTU1swaAo) z?Q~2Q2@Kl#U$`WAzZ|QiGd*IBvbtA7|`Cyf*KLr$^Ky;z^T(k5nohT zIZo2tk*+|KCsKlSH(g+=^hvjAfhK@U+r#kdvd1xb9$VOp8sPO@0#IBqw zi>bJ&_Sqx?oneH!)49Nn`)L8aind#U0MBdUaIJFkBKb<##uLE$%sC6j7ui;__ z+6M;i4Px;5d(3h4bErSGmq*$r~yv)JNyB-Eo1}Af)8$yV#L-9&E~$!>19$Y+=~a@ zpZ`ks;I#Fe;>BUm&*x*i-SKNqhrj@z+vv{VT)6zxx=J}ZIhpKvbqFC0nvEBrqfsm~FwiG-p>W}g>QDCY0^l;ih+>a+hd_n(2t-_WmVQqgSGDN+qUf8V*{Q(Hc)9afk1tGA5M z9Td`dnGDi9Ott~qWGvElJa|ubuf=zus;3A>D4L2-^EK3^{x6HU%dL<->v9AexLa2X zM=w>_4F+s6(g893*e1p6=?t_zaN7M_e>;;L8L1{d;n!~m&wc)j&JqSU6d}@c0!6DW z3Mnv&w&sZQnMe>BNkuKBC`aQyaDi`ZmPEUM`TVsVE)00W0`|w zC}zBNK zBhqo=athXp_E8K@_ymJk-0J(CLwdHA@A2F=i0;@T|0&JA;ZF_#{?($0_~7%3(8ss` zqR<6~B}Jwc!#A@+d(x8rfOkR76pLPGC{}amWmvW@H}&{?+d7TLR-)DXx+sftmI`Wm z6`)|Z)E(&4>&1HeRLe6BbsdPCWwHEVQ&eo3II)8FaSY`IXV7}7h?pohuthi~i z*fO#Cu26RF!OKkAUUkL&2-R0Kp0ByqG~JL)Qsq{HLS}@wdXQK`d36qxeNEsTcmJzZQTrI}oTIgEQ>;_mXlXcy%BaDSra?%EvIuPs>^q!x+$KyV zmEptIX%=#{BIX&lTPhbs4W%+!+0a4{XNS;?@w8ZqbJ&W|+U3iBpi9AtpDAs0EJY5w z{Jq(>>+5#o8;TM9J)sl0psqX_eMVkoEZm@|U*AgNh@2EKTd*-kFn##$I6 zFQ$BGXvSoN$~xI3#+*zgS~%`Go_583M-qDLGEhX!z}%E^!vr(Y;c|YjZ7r>YOhdhWA}~Ss>(%q7EoWl5l@2jqxjI7I8tG%+RG|mK^&{-h`3y%(9$a{~XOk zZkc1<-mb6QV{jF83Df(~c77!8=z=aqvep{ zxNJPxmFd=cXIJ!)VEndqpr}j!+%=geo%5`sVHtJvYVW}r+<2{*i&UpVv-xbviQ{cu zcU%OTSJDLLkyw@9g7^~>x6nY`ttZCBUnv`9I;g9x~ zu`F^?Id)9#m@ZX`FQvg1@w#>q(B0b)Nn_m%f)Vo3snfz_~E@Q%5%I&)1HA@YMQyAq*g?id}`$%w|}djDK% z$K?HG=jg}7+Qi;uut)90I>5y*nE4AR3H~I$fZ_GD%jJIvs5fT;xT<2lvQlL?&28x>P zG9aNyfw4F+d0 z>Cz?fdtzAW&7ht2QQ8(I1+9(o%d=gD%*E!4Qo?)Wa9NU_&Vao7BwkmSOM??}P`*!3 ze8ixA){k3$W&Z(49s^bf?iDiuq@2U;ai63#pBl2|Q@{<`?7qkVub->%7piL+7k4va zHcw2aQqFmYH`7^nM^fP#Hnl;LqCN$QGnl!qC_&H54YaC&akb}FsAHy^fKmP`rhEh; zRM*s>D`X6d2@ppNFh5e$d4XD0J2ZonYfCM_X`vFhHMEGK^ThLQ+@}$tPAtIjb3$KP zt_|y!BY%SVxsgcvsOW0^kr<8Uw1o=(Ne5>|Y}c1)Nz!%CEl~zFlLh7A&ECu(SC+$p zN6o=(NH%&0InBTNGlU(2WCjFb0MgReT_Jij=ei*C=V||eUXoFot$xJgwet14+N%}C zk{y!5hGIf}1c5#nLt(D4DcEm|I)P@Zq7xITX^!nndzb1Ds$S8|#cvLwen2iri+aiW#O(@Ci#&Rfb|VP~wsI9;D+ zcUof6E>o{xN4v2ISP%P~68sMD4RTg^V$OJt74&Y7;!FlDXGQ4!0-=~Nts6h3p-HA? zesey)?-WdbnKsXx{N-CUwy5>WP@|S=FuNzdI8Zik(L|#sa_MoOboP<3Q^~`+)+9*; zK_W_lK+=Ing@Q77U18zc87L6E^MVXbjNJhT`)sS zp`p5`AvaIYMlLC;)|VB>#$%PpL{lUvaoCwWc0Ecf+ID9Vws&Ji?bA%{6TmeCR!Zj2 zg;XlILId2C*`jXG=FR!rzOZW?p;KfbmCv}u1xm=abPkh2wxX6rT#%%#`fyKsd@%dg z^-jACzdLFnvs6j|}qXVFVl@ zR+_nJ0JrB)bli4+D=D*jWcob?q*}Pto!cKjtU+}JqBR}Xe#!UtMZWzlvtom-e?l52 zqOoUBh_+;jQ+p2isr!seA3YpPRGzG;aw)9tl)lzbFkB_}%kboc>@Zf0jW1p`$DF}G z9VR$dWAEJ*n&F*xCzc?ER$b)%xwqAUIkxs?S&q&d&P-fI7qoxIWVSxy7Z7In60i`3 z$P6#n3y~STU@Lu7Q6%^oG?G z{lJ5)0{G2{1JEYqTpi?iQg<8hz!@#a#FEbyFd_6Y@X*0`-@(w13t|*9%PTw`T!RQv zt@omC>&4~>y|#tkWGv6pZTiod&Lr*}*{OS)se9^qa^ohX4TO-O$=7Q3w2MVD5y&@E zsf0t(p8avw~_au%fTAG3PKTK3r?0K?23?Hk-&bE zQ`jQk?*ZqjiQTz{UD* z=RxBm>t~2#$C?eI$>2@O^*TWqLPN~;TH&lIabLnSGeEe<{YfF!p!*zTrmCXc082EL zI8afR{}?UUhpZo%R(Yb*6O|uLU@>VKY*0|-!g7@&L>Dk*6QPqG`H zfEUN$=uB)XY#|qbG$kW@#aF^~0nct{@IE#Y^M#=pDX#$?5t|A@Kmh~7hMB~Mvo={? z524exre!W>9% zo+QL~d^s!v92V_E)DQGbs6ddD(^gg*sVKyANWEao5B*K68z;)rk%>n<+<&7>KICC* zv&Ge2qJVJ^nomUfe#{}I;>uupoKp|tUN{zL0dBD~nQ}d2&JouV@#AE2BAEl0tl+$n zzG3Oa|J=e@gBQ;+ajmB#+JDZJE3Atyt77aXCzo?(0Oz9^$6(&P|FXLM%<$Z8OrTGl z%OTZ_9k>rL^VSHq3HueReOUMU_{~`wGQ?Hr7>m(X>)Tck#ok_tp_j_=xIni>d&SH-r$foOZy-E9{1AvpF>2gTdLTreh1wE8aqZVuuMGxJ06 zN1$VP{vyow%!|GSNX_2c>wQyKMb^hJUtwAkhKy#?UML-w@X>4*VyIW~3=9F2Pl1ohYdvMsJpSnsLoM>Bi6P)?O->$^Y$T5_s zkTb4>b^7oLf0N9=;`}aUScZf6tK~Kv%}r}e_acIPM;nCOMerAP#{WgBJ9Ld zE!jii2#nx2{e@YH1-`E4C#&B{N2|Fyg^1`#Qv*ZH%~PZhEg>>vLO>ynV1Qk8Ko5*o zFYX1|3M3>5d^BBP zvy>4`IHf}uc50-KK08S%D&G1!@qbp{TfxwY{PRkE-si}xhcj972lL%) zzHI+!8v$-_x)-z^P|`)ONUSsR9A4q#k76~QK@F}EG;RUU^N?p>Pv850CpS04_5j-> zs!ae*rg8n{$8P8N?m;|{6TI8G8zML4pZtF(Zt|Bnc>NL&=zlKhcQ?G;v%eYlJ-;aj zQ`@7d1!3}bTCaz>(~|e}c9v&Hxs6ympDXl0Iz=01JO&)Qte*w9@}#e3JV!19E2jT$ zpEvS3^ClBuJ5zd%O7yJ$bf=?b&)gB^>$S&+2qwt?B=AyP!(0t8zrS~(hrWr+w++Uy zYw?qtwy)CRaNyjNrfXHebKT!?aNhhKBybG`CFJD3j6Nl0b_$oI6U``4Jp5@yiPDA^ zP&P%e)}M5iP|Kvlxt~~-v+p$N`z1BEm4mDasT57W7QX5I46R>_++oMBDL6VGeI*FV z9jXA2nuVGI3!Q5W$jvo0zwZB@JP_Ls$o4#K0IdA?Ol*o3as9U0-u+X1UL_h#P;GmU zmlh%RfU!NL^=gb)(J}SLe50!bn7XYNq$RqpQa4kMD&useu>W>@gew#KeOMvZY5>Y> z&V@GQz684gF3w(F{$K+5$(5L4|JVciQLs1f4tcgkOvUyv1Vs$&?7AJo5e zF4uJe3@#DhtYmoth+O`u9Vsfe=fC^EXxDzv6BML#0-0X`Fu4JMgOd9HZ$OjyR^S5o zz-wHfcEkmnL~tUEcn)BMc`&5MzDRhYDJxPTTwpI!qQeS$ZQ#O#T~$^h`91$_;ObRC zNb(|ts2;g)%GNsF7vSNT%;)>4ss6Fw?l{vBfOy(5jdVN3y*3ue0nz3F_bJEnqt2^q?Pooh)gQL z{@4gGoP7f1=ji7cnjZzsLA-zdlgQ(K0yxIE_SHXs*}%<(8ONKz=hTF8uDPR_34X~+d1w+SnKK%%>P*BFVC{i6J(vFW`SQ4g z=)`az&ljxVeGJyqJn{Bf7*p>N6JNDvsVkVg=kKawP2N^MNDlVAb$#J;s#aQ-)wuL- z;wIzjO3dV3uz!*q#Ir85 z_MNv`?IGQ2(fUnL`*byovnr$bm_l%UX_Yv}I=x=JnkzEdI$h)%Hx8K|4M4|}AkY(o z9riCS=lS+9J&E`#c5jxM6FHm>HO~P{yA)!`VzHk^-gs#<6`Cyx_ItO~oBgWWw zy99l(wH?wO(KY1BjMi2AnmGQ+ZWV+$G*s7Ov4(7k_h#IxZ4v3kKqbGCHepkRklCqk z&2^;vE9Ne+=3cF4{PD+?X6jKR`8dAARGwloD-XMvdX~C2vGUScdhdJ+ht4<-BH#NQ zMr)#Zaz&V7zalJEY^uRxmbEmxLMx87YCZ!%&-T2j|__jPfc}b$cn@x7`0*6)ypa~ZLF`?F zH(jTmjms|Jy~R&U+RS2K%SgOMzr_<s8(^eRy2vgd%9QbNIde+&61+i zsU(Ux+2E-#o*55g|7u+iyXK}nu3NBvJ+6O`Kj+W?4)#gLodcrnqcWX86AbW@Xf6xL zBolw5s(1~ZHnC92^7L-wa*}H9AYxO!AoSHbs2JjSutaaPtb;b&NJ(ztsy>z4=sb!( z>BPwz9fUo@-LxQXQ@bcOp9Kg=H&~zY;oY}-=4G1S-yBkdzG!>Wr}vj=<1J|YWf$}z zh=0`~(H3j&@EBa~-$pjz%?~ZmcIg-VEo-Aco%?E!u^-ObH0mM)ZrhghC@l$#Pentk z?gV1FYT<5OGmXz*-K+$3+NCj^WfHNDd(G?8c(sR(%Kz6U|7n*Gw~*5R8nqvn`OAlS zu{ze#knCGmj{S*A^od}Rp(kQPW3YPpu;cFH^|txDV1IrGuqW8dw|Kvw?KySWZ!z0A z2T}ZMd>x0EA-ebFuGVyb!1>RDgoc@ECrts@i(9a<2g9E4+SZcOxesPUy#CLVY^fip zbRGf=`rE7}+^37wfN(nw(J8rdupul-(U|CzaDGQqMZ04Sdsx^=n}#VGGI?9>cbUN9 z39K+T0PDCl&m(E0>9*Cr)U~QJFRMX+W1`+*WkGGB+TmC)lVfu!9fh-rT?TWO9aEe) z60;uA?(gOCasMb7@QD@iWmc}aMpful-l6oUpNrZXOlb|90nu(EXpxbOS^`gZy} z&M6q+|8c}9=>O@2aVwyd!=GQ2smXdPGDFK%1)3qvH#lre8UN60%E3XGKjmG_s!DOM zU-!wS9JQ9d51A~uMV?*m9ER1@8(c``Kvo!eg?46hQh39&0V8U$vYUEuq=9XK*>0n+ z-Rz_9ehcJ-j4kjq%R6E-WjkCqJ5gmo-_B0TI5E*|`j4fDtN+uWhn;>nd+t3-ei8V* z?5J4vr%t`pmURFcc!mP8l8_aeWUC-e;Ngi(t8Ajy-&yBI1Mo0v#1W;T4bNU&CSH|y zvlEKWkk7uDW*D$K;%4i*i&J7{8{$A~u5EwC=K1^4psC1fR>Jm#v?Gi45qT7Uv+}ZE z*Er?qZ0JoBv9PoLCTS9y|H8s>?)&Sy*$=x)qX(8u@`6uL#YfC`L0SC5N>J9OGxn7+ zkcvEoe;RREN3Cq&L~DAR%An`F+imZ%wGZl<)a20D2o7j`-n|yEdZT2y&0A_{6;~C{ z%Eo!EIVnF=gV`biYQu8qPU{*g@=O9dIE&um2E+x~KYEb;eYqWeQRXsyvxlSu-8nfg zAEYa7DT@VNt1w*1DIK$?vM$6IL0dBT(&!#+3Z}YjTJJi19doBXa$bN$@4wyyP5Rn| zT6JRQNXCkqerLsTQJl=`YARz~%W_3LKc&>3s3-%Td64H)ZhWO1{K@XA4Qn(zQ;k5Y zzJ0hVjNe*ve}+=ZsFbJ>8mxu~!ZHpWlxyPOJFV|QeMKyoZ|%SK*(3SsTlS{?_(*X7 z*m-(3e0)afH-4DMH+=@cdS5{|SAWY*TzwN{n46n}$owE=oM(LcO2GBXvpxYx%{R_{ z6}kjmIo>1?f|!GM=l&z@Epweq=$(F3&Nw9HzjS7(x-HflT=V^@YRJvLXZ1+snIR0H z+o}iHd!_Ix2#U@Xa=+Oo-&;II&dd`O`ih-{mZUMU)(CD@Be?9JPgK-?t|H2dlNk&a z1i6h5-!5lhfZd`a{0)keVOXj+QgUYlGOO&pi`^#`f8+pBO552`L}_rr{g9?-xKc~b zWfp&`5riUFncIVvnXH9uu%@lB!hU%h4gc0;hLJD>xW&RP!mF{UpjpzB-p_qo{d6Qt zR%17>z~P5C2BsAbGilVBo-@UQ;2!}EGbv;fggaFYGa*O}E*N7qKgGCQkN&^svFD07 zrf}x4^e%3dfU&?h*9!P=GCxE9tn=Z2>&)?2X8UiQUSA}FR3G(*dTjh~M+WnOytgPMWlcDx~;}W|=A$ga|w4WoqkadPN z^9{s^vNeh?dNrIXf{f^wtf_AFYiIWEnxkE7Ynt97mjxrerOUg z&z;%3%8mX+7RDMBYZe5t+*F-;xN&}{SMdDKN)($mIf4sCemCYGYAu9Qz@e*??$<_) zf@TdKzy0d*u9}DYTJ`d~H}z95w+8!`i0(pAE;k1^I2-q7fVR!|m?}rTtrA>2uD3?t zYc{t3@0T58ER6!Z6UzVpzcX{^z|N)h18n|xm6*E(J5Y5CxG;o!FW>y=XHZml`ZCP+ zyx0Y#X8!U6&gcWOjbH@&;67f%?+2!9u6jZ@if{tgGT%BeMcVq|4`;f+F-$kbsHOVjaeJ za2B}>TNiK6G1qvY?q4|59>3;_>`|+(EwyNlhdf>1V5igL9cAYgg&!N%vp@gvBG@eb zr*{ATN0UHg_&EES@ci@FKCbu0CgA5luvj1WjnDD!U^nExvxiwNh5Hs^d>qrMpz|W! z9wx*?W4kvb$I_!WU+=K@B}H8*__~I5Pw0CQJiqurD4bpRvyE^AZb3fLEa=cH05XRt zGJ#Uvy10A_oWmJhYFhO4=bX1aML!eE9jy+iBUr0wYB?&ru_HEDxpJa*5Kut!^m|08 zfB`Z$I#<;sf}#ZD>f=_s5JQ)H|Pd%sCc;d{R2w1qKL=h2QuhL%bR8?P& zRN~gaAPCW5ery-A5wnDCQOMC?D=VAlaym{dqc3wty+I(prKEE_rW836Ikfpj*3W6` zpcBMdpl%*v25+0)aYT!hvz!7Ne1kmds}s_6w~AqziRNM};scQc)BufKL=vgH*|SQ( zu4SiY3B$nP{YAcb*mA43!iJ~=H|K%Fz`y~I`KOXB4ikrw z&A_r}qt44%nmVA1%^*jPh9(P^fucJW1Jp*k#WTPNE05T5naTfOh2L5m6%g^!6w@vMr z&xlu<@c`R{%4h&p_R($>Sd_<|8`uPaSYXPC=SXwS#>(C#S3=Wj5vjln_hGT^T5=;H z5po;@;+AY_!s{ngcdY2T82$lrHK?9RAqUq3zoQ>K#I@~R9hLWqPsl9IY*=;uMzm!C zj=u0NjoRPQvOYtOXFvYuxXgF+3K`9dxNA5nkNRRN3_@qf}8mx-RndK(#gZCyTmKXrb&Skx| zO?|_=qHjZUg~XHX$g~JQvH6-$!tyiB%}>H~O3-V-?uhCMV0PyI=GDgyB(eADBif`L zd45$62nBt^F2d2A-U{+gfH#meS^xB|c1o^7F1dmPG?$3<6}J26A|+_DQZ#{}5N~nj z@I4uBU(y9Xb33dNy431cwh7c;ij&^pAH0TGHMEWo#lTign_S;4ohD^g&`XBW3}yVO z-R!o_(X|d-y9@%M$@&tAF6%Q-W>@Su#?g^^6=7E9ztGI}B0o0mw}qTu%A8(_-D2aa z2=l-E#(z_-3C`kH_~zJIZ=qT1MIWh7|C(D8WHtW_YhEw+LVNr-XHAe*`Y){XY?05M z^WEfQe(0VM&w0fSseB{H>1#UnH6eL$bX1z-`{+sK=lwmdj0H%|-QS1>>_pptRe6^l z3A_;96w@`4GS3rn3rrGzXTyy`sYlSb!is5-tQh|b7XQt^T1eu9PxCwH!n9ywSx=iM zS@sZfrekJl{KD*ljsdsA#3!qkl~FfMel4@gsZTH0O9ChZx*AqFR!+L$JCH$$ zqbpsD9tqb7b0fp4z)0f-(ig2E9J}Ejzo~p6Q~2!?$+_}yi^2l+)|xn_1Q6M~p--T_S`oGKC?rGP21DMi!KP|| z8#23;0A=GYHnUr5?NikFjFxx_l%L5gALe)b8HU0&SBeDFZR%q8t4eomf~+(GMeph zL!_n>VdM4=$LZZnY*SzZH8c$I$A_9`SnwNY1j%B{uwqJ{e$bpLp>D$Z9X84yYlk`p z;)x=3TJc5&NqPU9g{xG!Ov#e;Za4?6eq|sJO}Z50a1!8ha1>vxhzW->E$DZN&IJrW zQ9vpI_DKNp!DqIG9ShiuT$lPp zgj_*q|zke`>OgccZ?Sq5|+#fHLAdi>f6@7Y~DFL)3)n7i?~g zxHMbhUCGu7B%Q^ZbKnj^T+HI2s+jq3PzXGO^m&5Wjhe5Tj z0o^@({M@rYRP2v=65xQU7zSWu9^V43;1Ars1FG8ezxKN12cn@pZCbw-P0y-m!RqB_ z^p2ud8AR~LN3GWU!2h>#n~`x=a^}ErC8MW z2I7^sj|9ud|Ml_r@4jJQU;x!xoF-Y~KHy_I{b+|bkA^A$J>2}%0{8%_J5qwN6&OK5 z--w|@WJ=Ld4B51Zvexn?Anw9OUI3WrT^$ST-_ zC6Kv08KJPECZ>}~<&@XP5m{1{(Pi6B2o2dgMQh;_`nzV84G1|60_sAZNVHi|bxn92 zAu;twhwl<*ncO~8yvda%*qIj;wk54jO|ibb&rU}W8S+se2n8Xg>*6>V>E7PcgX6(H z49hq+H6+nU6cij~^z++&cVLO-ggN%*kFmH>_e!(}0l}$xCzD#Ro-eHAcSy2Gt_;y5 z;L>DvI^zY_Q4i6GQu8wJ;h#{izl9{uSd@F9WF-CPB2_+c< z&Mz?43`Jo5tpyd{_pXXRC(KYVlD04UXrJn>zVqmbH|qiL_c-_&3P$<>iunXx?^fYyR*kP11r|L5^^;#N4p> z9O6WCBlKobM<#u}vdeNsCrbvLu)-W^6B${NBw{9>_P}#4aZz9_5WKms=uLP>x$$tT zkduPR)G{7gSJpxpE5tu+hD=B{HO zjN=I%|v1iaRHRaFRqsQ(dZ#RT>w5R>3pY*r&2W?Yf zy8;fqpx1&gvWkBtbKCeQ_O6iloIQUAvFMAABZg~BLt9;j^cFe44(3~>obI@a66IL| z_Ilygw6y1T@XK?Pi>v6I0J7Xr6+;V~BTJxk;zUBr9^af9sLzIU%-T%~egfwsxlQip z$}Nl8aqdA$#a8U?$%>vZerdh^E!RH zvTxJ_R~3J1;)`EU;r_R|x*V88!JB`c5P?%QwJVUcW4?ETpgxZd*_0SB@0T(JJrPl~ zjDyzKl{!*DP}tpVkZ)djfUza!<5YK|CzYVBi70F6gLcXJq@CDPk=2|{Te7~8SW!#R zVs)l9!Yn~s)9#`pVvsU>qLzM9&FCS{y`#kZ3-Ut+RIgYT@9~v%FeG;pQ&3(R_Gwbj zx{G#V580#VMdhbjTe4GWd8G+-m;JKMW`o{>nOLZBx2(YU%Xtgk53fGSy~{LNgtO0) z+`<4+;Irror^-2E5*WL7PytLtz;|gtoZBOGmB$h5w{-K8L+$bL6prN>S z^ik0^*rLlv#%$|I78B-%F77 zK&+>YU`O`7+2)TylMv1e$2Dg?IKyP=il>9o&j<_=5DWnM@LT8rQM`GH9)=oI7%QPo zr~)YEHYBB=Ae)EpvXwz=gj+C~5pMZnliKFxWdD9}fsm+MAxr>dFFeC{6zu_6#^6)^ z(*8e6?5@yk>@k^v2rN)o-3x?-1NK1ufyR(2I*QE0ZuqXt`7mq**SSBG{!j}sPEwt6 zynWu=a9tW(JhCmY^6|Htt7Sk_9G>80 z^aGH`i9VTAqF=9+*dSboksw!lq+^XJ9f9(Z4!Q#^$R!t_smD?1!?RSA+Smou=YD&9 z$YRx)n&i4aULi|*w$%qIlmc=_23_B>2&~PQe!Kea+sSJRLY?^_&9e>QiqJ zyhEBpU$0e0A{FIFj%z>6?4CZpzDGARKl}G@ExrYDbMl}2adPzY4B;RwEY2tmpA{M2 z%g+phz9}FEEbx}<-(}lV$?J#{v^d5Y^T4h06Tu}?xDLD)@dP#739C3Ma(u2XUNeRw z_OKqK1fQ^rD4e*RJ9};foWgkT#ZE$H)}(i?{o(p?htks>9rrjH=OfOqN(7PeYlH(l zy_PuV=NAvUlNW&qco#-VN4R7i_i;%c-S;;Z_($Vf;Rf#`8sL3_I}-iWqHgS;Dy)|O_DhVq-1;sM6$@~IlAFfZ}f65goxVy zibqb4_PkTM&FAtKL|g96^6!>RF@Ys-(eo-vCO#ur9pF)ztyJ%WtDXp})g+yy&Lr;7TQIbh%8BctvjMq6be(GN<%Gf_ZGB zC~gk{mZDW!c(h(eQM-F$`vu`$6_h6S33NAauC3=@GCspp8jShKzRrBP#9=naQQP(> zI(EwTQbZM#loq@a@5p|DPCWTbZc~yYn*OLOLeN1VN}WLrrOqm|JQ^?9E)SWint9+}>A$_qm2DV@<#WowTG#ET2Olz6I$DqAQzBNRwg zf$Lx-Z3@USD(v{BLs1h4VCF3_`JgjojtE&yQK|0XLMZ+)%ft_;I?yuL%0<@j81=Pi z1O6)Q4aaJx%Bg{k7dtca7$p;1)NhlQo47Q~g`)1Z&&=D`@rb~G-N^Z-n4dQ1(0S@` zu-th(OG$=WS$WjcCUM_xO&J;(`$&hW+&sLQsGlHq<*6r0qc3&)QSD9Qb9%T}8@WIP z>%6&6a5bjX)-ji-rzYwuvnmgtTbr+G4VlfW`r3~x17&h{yGR;r{0thH7GB}p7(1>mmdWsdztzBci)!`D22$u#+;(J zvY2eVzuTBOu(jbJdLLo-B_;aA0=H`hvP4AK4ha|zX~1oX7wGA#oZ6*3bpIl1BR*A8 zqzmnb}LSy8++(AE;@!(MCl=nPsaWs4dk9~M*=o}Dg$d$H)*RulKI9N{Y(|Q z^aMJt4jy4fCUXV}hl&(gtyIGtHp(Zabt5~%I%uT*1(MJ(&E*>6D^w)}$07^bKh>KI zQtN+9{z?Eh6-e7Gk?qYwswe?c+rUeLQ-uIi1%@go*x02Vx@5+tWL2QUuLRwa`Cdbn zuLP_x*&gKSH;;Vr?hU~uKHapqky0Wex=bv(oRB~vvZgnIhOszM5@hu}5W2LZ9wdu8 zA@tfNH(~ktU$WZmA+18&5V}5cj9YG~w|0!-ZeVkMJ%du>}yO*hSzb%ps@<*X8H4 z_4T^72uhrbbxx49Zp#HdUQAg_5Av9|bYV|grcDF>NAS@!_s>iLezbmPuiujZXMcCEqQ`#QL$9n%0C~o*+%Iv9fK7v! zbluTVqufqI)6q>nPtA35t|Kp}*Q_FpEBo&ZBlycJ=8t@Sii`^jqs1L&&ftJdq+@`j z?Y^*}u`X3*evh~P&C+4Sfs#~IMc~VlK4N7837-452Ur{ps6Qf~V5y#(r$~t|w-(3@CKezA(WN$o4%%`)g&5B>v{>%;lT2;N3W2cro1MYxN43p{TXS$qj6H@qhe zd5MIihC_UvcX<%L<$NBhd96hdvoaLh_vUYf`_~2I`X=`B(=I#a6I*i?%Hzd8>f$mE zo{C3KT*6>mbN4E!ea8dKqOe%;ObRIzUyr(;frT$Q*Ymz%cC9XDok;~3qgvm>G(>Vx zm=qrbLHVa9o8XLVuuyU~++tJ-ODm_ZK~jS?uYOs_##mwSLwh7kGmlfWcubz81TKm5 z3G$ve(iE7*G{Wi(Zi9l~LQW)-m#~P7%mc&morX(`()^Ss~jqk1zY^(U-P@_h)=A0$Ut_b0GtA44yum&s#Zqrnl})5nKP@!=)}w-Ib;M@nC2~(p4d! zQ!7IMb^WH7`cAu>hdP1b=m$c#>72_`Yi*1$t5J&vM6DAQT)o8!*4kaWQx$KoU70&h zjuTt6VArbnRM0vA*Kr+X!KRq{l3<;xR2}+JfMRV}`H-rz4?c4Ckdd|2t#QT8mVR8K}VxtRu0B$opZvAuP@N@p6QApwH`G7pVoP3M<0E4`|{ENr@Y)|VPz|2ge08|xX zu>Czz#qpTFfOV!e!~4t$a8BpU%o}UN-RKvG1Vcaw)i|lGNW8!A6H{t#jhC{v7dWJ` zg5;sG*;-4LlfbWccBT_PAl5vnOb`tt_@_6pPCdU#>EI-x5#hnLxRxX!_n@cj2WZzp zv-q#jzj>9&%aJfUB8K$UP z=M#~HlZuvK{p*CTV1I2UV#rttwkqJ`Ll529 zZn5Bhbe$7O=~5@_klR6Vz*V>Zo7v@wb5Y@10J}KvO=|pz#v?uuw@RDWZh5#We0i#) zxAWR%2=sj=gc^LsYT*0sXy({52;Z)BdKbh>E0A>uQquGL=`J5{4>&k`pgo)rfHEj* zQ7mws^ENsJt+;pPh=(~AbplA7rE3zzI-&(~&*_c-i4TmrDezX}OqP}e7WHpjBPowh z0u>&q?A`)(F+2YR7uHj*dXXC+*}|}yZqJFRmRJBx?K-H3355!<;ve~Rr=|Ud;!vEZ zdrlD7VG5YvfKw)ih4jfl!=UC)qGF>Yn`K$j(b?|y@?$V6URCtbC} zY%X|7y6>f&*9Ov&Db#rm{rhyNKWJN;9GdW^30Qe{h3-Vo?Uchu$8_+jkr|7>8-8J7d(_NhuFQ-rN@{ zE=F_pot}@#iV%I^Ey4vu0h69O$f}Bpv{~HbSwhMRPX`Go8-m%HqB8~gLn9A3OQs(p z<=hEindS2P!H*V9f}!lHPV%k@xsC4BF}xoVFN9P{Thr{`L31Upt>@~y#I?TEk(_fC zjJi-foA@K#bvyov!CwOf;82BRtGfcK&oze!RuK=l&Cl0ISh@$4hYV)M1NJ9WdA|h( z$KQzp1)v(@LVz4d6h76NO^S{rN8d1<#YGU6V4t>l1Ja24pPmYXL%7uv^FPTIFiq2i z{S3{G7r)x_WRGW>C|vut-u!ajc-sEn$7-`3EX=+N$~Ipfm-_LnD+G1BI^N7+;%T~d1tjkMBxDYccvy&i@i`CHyd{M9nmHFBAoVNq9 z(%u)^4)LdvR{g%1m+g8Y8P%@AQ}f&2k4&(jU8GfH4RQHEB6?kbat4j|Z?hHZ{zV)# z_O?{j*T(%yDy;sZ5vNn$A68$ZPR4H3a~+PN@fp&dn9l9@rMezQs%FdVSRE@CHb;CC z$IZw9vSZD21&2uW$4G;&*znr8yzDmQd~m~ga3-aFbQ>(8z76R%ToYha?}ZWH(rpcn z!yF{YE>%YJJUIkJ;a^#ywu;XGgr-)aY^08n4YeidRuuF(E0NYnHQqrAGCRubp{)Is zR>D5a4n}bvcumrs&a*0xkgHKGNTHLHsOH0si;)FYo}3PMYmLz?o{uaSfEO)IJjBf7 z57S?x2VPZu$4|YRy409N_d*f4+&Ki5{yVt4?xN5ZJ%jt^_QWdZHu-929h1|?W>>`@ zS4dfs1{1c>Gy$q2m81oUER8XlUy|YZ6`=rIX|8n@8gd+2Y=PlNga$xT5=3gU=5MmA za|YiteiZX$?x>riCW05y*Ef-g%2`QGYaa1NGeTNq0au1e6Y**Htz~@GUh|hfj2H{- z!+X*#I#*>=f{ScpWrLbTY+EWKa)IrLgDso?W%bjR;L&|a+h7iT#h_L30h&(gP}LWB zLLxW29hiR)W-6F4yIAgA$ia!M0B~MYY-3y?x?DC;i%KBO(lRV=9mIL;4I zW@%!q0+?It+Y~+C^9EtwKe|}+$1#Kn2K%pdxCihGH&lk%N=5feoitLXx?Md{6 zu!!eda^pwMpI}6q#~rr-EPPk+M_8cK9v?CwoO4V{m%7QGM3B5eS+~>lQ4pg;UKE6k zu4_S<_KMOIoZ)>sr;I8Y8OIp`(>blrskRFNUJy4VKiFncB2?mc7eN+Sd%85D?wh@F zMSUG?=xJ9xJz2xSRHycOQ`i!Z*Rob33VBaxtP8JL-W7gsYgk?1UzHdGdhgcr)glk* z#q6G0r{~`u?QJy-2+@weLAa=_h{;C7vLp4>Sv( zHEIxRpRE_G#p^M8b)M!>dSR`4TgmI!yb)!dPHPGgC&UpK^3DwIGFL5ud+y=Kvwr{V zys&=63t4cNj&osxJF14ggbTg5+PXP~Q4@*6jBB@U@`}UGs_p*Kg_Uw{`if%OPMcS1 zlNu&d(C~|t9SyMhsrjCHJsElUv>MTjH2@-7sPojM36Yw997JJRkWvs!uM1I1GfEm* zA>S;k5ET=08c@!K*|O0rZ9N zsy_28KnKX;CH0OUsKB3T=-UdEDQC%`v#+mVc&cLC*(pcuLyFy6nZ|aYB5wjaxT!PX zqDy`|q9N8WKQ-c8#DBj#kh9aL`(Y&h>a2M#U6xVXx@h@kwlwSK=>qib`@N0Fu5jK|v+B^oK{)E@4U)NFx5H(PL>o>Jt}&plW&(|0b!WIc{?0YH9)Nl_fSiq=+h} zqQ@4GjadBtG4jH0uN{Pn>T-(~MySiy#!J(={?PqB>$2n+x~eFrT=JXwrW;XlsqP}- zc$AV>KRJxjqH-g&3%Ad*g`Cu0S<&6tIBe%Pi05@{5xs!`%{^C)>qRO}`BKP^Er8Eb ze+Sq6djR7*&$k7sVicg=DBfcjl#b`+*8=19zg@+C2j_S}qGTl-$RjTpjYxOEP31?s zQC%{Eky&@z4j$zb^p9!V)Y~7DtMjAR)ZB3SvDkM6x|e&^zM2%qu)V>oSfhqJREItl z_Jl?ypk8gtXdgM%ka>SI$6T|^KZboHoZ5%ALN{zob4Uqv0sPKC$E1>tm3coZ0Nztx z$I`_y^LX*HpFH1h>=@YOJJSj5dcm`L(z}Q~{}(Q6=Uttc;2b+9M;@%p&@{yY4qBti z&Wux$TxDQYT~XGiDIpx9vUVolOci(S-yDi#{R@MYk+Z;mc*jUQ?Ed0&mp+kpTrEk= zInEx z28(P|Ty{UH?#dLdS1z;pid;xf!@mwtTpL_P&^LKq9pZmV#B7Us=^Kzdq zjNe`<03C()f7ce0{-KBC?(9d{Rh89C_SP6Xq8PqHo_v2MtYH|8!Uk+HGpca_ZoWX%LrBYta(f6m7?=NryO1F&=S z^7P?Y?gcF@FyBAVB=UIf1$iDMCBJP29xOCb-%6NuB0lkiCKB*IIu(8TC^6olx21Bi zdeM{LQQPdU;sw2$=LghM5nMy~`4CkpfYNwTRkbdC&rA+0fktL2$u*$uGgYNK;dZ z9<$Se>rifh=NfsiC|Z{qJ<1H)Y!k-N|WL?pka4q;o9FuB}u740F9-avwNJJXhjS&%zx9Z`!h&27n2`zKI;iD&PW%hWg zkC@n}z=+*=i^yz?axXV0HSqEC{clzzFVs@o7ER zOhYq7q-qNkMdI!F5Ivbh_^d)nSk}yA2(IOs3gyPleDhhB^?Zw~>HCT~f?Ee2MIeY- z9+C5P#h=#whokFCrZi+?xPCZ^Hv$mY`==5mro!;jkTBB5eFJeODjVmA1u1`D=+vT{ zNEqN`mEmbeOOH`(Ihf3-7AIUMCZ1Rc6!$*nkbbllH}0_0-2{emtVTsR?PsB?E`&JE z?U8C8&h}(vrr{`1sad>0G~j$<8*xXL!J{ZzFEVQy%gF>UvM9QTUDtK7^s&fkU}n1^ zqysTwB6r6zSuCNUiO6VNVV^J|B%ZHm7_265aMbiO7wMgg)WWdaAlUtc^CUp1-Bb7m z$B*5$og$3jx90FVYOgxGF~l5TX95D%f5? z53)n{2>zc&#UyY!5o(rBesmFSbNu{7b@9yXUjH1_X%RZbFb<6|x#X6UZ!!`2vPyX||(&hd2Et>hsz>VGI z8r1{wP)jvFPc>h~*XtE_%9M@_>U%v*>}GLxo*Nz>m@%VC>S~}7_Q3*KE?T23Jhanp z9f$}7_y16y6o51t(kT2G@xN*#B6Q}|ku))R1mBdlDLBVoh>(~cvwvNN#4l-JW*kKX zY(;?{|9>y^II5Pb>_($$QrKoc-Y^^O`D)ZMRv9CaAO+ktMksEyf=29}C z$m-*mVVHZ!@x=K?Ycc*gw0PJei8=HcQFEIijBAJj*Ut$=_6N&gCImirITXTPuu{|v zQARVFmT@r!$vO_~O#=-1-;$ z4CSIEablV9^*P$DT(w!-I~Pw1bL74uGymsr#f&5?BF5Qzr-Od7{9j*S!ypf(_xFOj zaPR~w8BQ-*o79dukbvxP@7^!?{>IbL(y3Rv?#>3;Ej?Elv|#gg>Y*-RVW8$jiVH$s z~@q^WPqhyz+^tJwCc2=yJg$KR0cFpHL)F z^r+@(G0H>7AtJ#I_FhF;HS!QPwyHm;i(RSG&Yh7-Hm69&P8Ee^LK!rRVGh5) zAEEu;^=X93hOcRzzH^7D3Q9yV5Y?b&hf5Fikth@C)vr*dNR&w-z{F>t^BEDz{ZuT- z>928o_eX9;n5#iD2ux-Rm|(gBNrT`->$mQsTF>;(usL7wK#B*LIJ9@^PEGa}87J*n zpR5rmJbD6qd=JPS zp@4TC7q!g^jVk=LPCxpBZa7ZL=1v=XyaD7{e8kiW%h%=KOUavcAS`Kc8Ahi|KUNm4C5Y&jqEte(ET;`c7jYytRH0=R-#VzsWsP!N2a z$mCz?G^UC``+X4{IY=WL;=71LG3wtA+J`#G&S@3ml})|&q5Mi3v%AVkPR<=ltor6e z%siPG97ZopM{JN{w@uune$LdXZq;ThG*g#oupATfv}8Z_d&%~DQ3jjp_;Z;9MN1RX zz6gGQ^&zU6Z+ejKM^Ld5u8g;9b0>l~>6nR$Bp0Ql)I{0@8tDU(%wYlp8x zMvm_8{&(uYgNuW^1NzzRL9mw?@i%qAX>ru-RB2Hw8Pm4Z6obY$nx-9XZ=1FcXgj9iBMe5 z7w!}|%{o4|y7@O|1r|?qvWI4pJw|80SGhkPqL=XN<$^I$CxbWe3K#ssU63I2#P((y zIz%S|J_gA@1swHGX&zQ}82ra91i8t^foKF8L5GLtjF=8qj`p719#21d=Km-Ckx14x z!NYMKM?Kr=^R$QS3b}TNa=R||b0h*tQizCwqo*%bX>yJ9u!FjhZaU}<+(%LVKOvC3 z{~H2v!RHVu=$eVn-`NtaeNs}mj8==k)`_7_=yrUDHu*vZK8iV>)94R2oowzf-wk@tJflI?# zQn5)}$*kH;*OHiRXjCSHYNVBbMcO$jk5+#}@^1y_Hs`NvF|0Rg0&jy$#B%eJ=T)c} z&5tAvL#4Uw7TN=Q9NVe&9vfQJ#4vQr-sgaE82Hf$a?RC1+HkGE$4X2u<>R1;F!5iF zJt9I`qn6~A>L0~%^OKzC)0{sHqTGLYjjU)!KuO_xXiwfdpP zgGimpn(Q*uuHs}l?T}bE4FqL(FJb#~rWWA6z3O#@kIT&kOEozoVNc0k*=PM>y4Lk` z2>xMVdO>woaN!A0su80>>6_=EO>mJYbpZnJjZ<|j1_G8V*<4_N6jD_2rV#os3(r9| zP@Z8t3PdqTg9lLz_-e+dXW!<*8RNC8k*z*WXQO~TXI)k;dmSO`yHZhdV%7AazLJZg zb9d^M2%0n89j2=GDjf2U3TO@RT^(lN6~QyM7JKeOg6dx)ZhIH5%3tdTTAg5n|5lTJ zqmS3kMaZ&u<(!OQ7o()N9iX3cHt7ABqgUVZ7(~=#-JU~mZNZ_r&)M9|xZjlDL*_{c zj5l)=egSm>{#2Ga$RhLv9zkhGr+#Ab3M;Egi+Szp5+KvdP>oY!55HmCxliewxr!PqnaFJfA?gK$s$Jbi03- zW?|uFWgG_M^7VB#k1mLw|3NK2giiSXYe3;%a8=OfD{ql+@YBO{{Ylc7mmA^;&ef0M zV!$p&dX!beXefak_bPqCIXE(N@>RrK0qem41Cm;p^`IIgaWD^R#t$f%c3!+yihlW0 zvkFO_-EmBZ-om{$k&xz5{VeK~^qQ;KsIXm@msVHTG5wHa{gNkc5hZRFMRV577L5L@ zTS{OYfYBy?nh7Dc4?o|A0gm?-Dc7xMk>u?;Q(B{)Hg$$G>Zy_k#~-JzF{j1~ef7W3 z`jfF*uwsRRZl9eqY|k^yVWN@Ihn72vouHX()(YPT@gA%m=1$1@9?3A7qBk-!=s1w= zFvD{#MR(cLUyxmI0kcSjRL;DF)QKjT(dHi6jeSAB!Q?6!v*_DX(RN)WcMr>um9U() zcx4r=(!r++nFVNWVc#7no7ZlyRN7$Z8(9#Js!^=`MCc+_7QPO&6POFo z4{Yaj`r>S%Mb|M6r`L$jp`#RjSTHV$bZ5>Y1SPIpKZ>}`DXCpGMJ1HF`q+pW>|KVJ zD*gPef!$b7%J-P7b%G;ygzNPkRtNh>3|{42s_2;jYO+4S}Gf>+Y@Lynait^gJ96>yrs+9p;_y;5zw&>lX6^iTqQwk)B<*&~1 zI3{Sjm6iPM8U#C=Ca#eA26^7)hwj%FaNL;Z+XLP+ zDraJL3^CORORGh*?B*5jo9SS?*h&QDvUCe`(K5pzSu=)zyvlJ);aVNpepunEQMdK? zVM%|JEN-Mw7tetz{cm%8_D{q1NiiV~y~XLK80#GegsTW;y$ZaY4%_}BI#B$&fr&v08^z z0{R%pJy&3LMu(uL5U|A!`|QmOo`jrV=% z^BDRAJlJ>=pixnemrQB;K5om3W3pTX2L_1512b#nYwVND4^CHRxwG?mOi#S%x$vLe zq_N1~;~~+K?3PLlJ;W~SmD)}v`nC$qWrBDEN}O14SD5Niwg`JHw#t+qpuql5t|+-T zgOJi<)R^tpLENhuEqfIt(;FyYbUE@Y(<$>%=K9`R&QJA_ANlIa+rez(CrP1}x@zP8 zRD_peoh{6e{abmfWfXK6P7CN0O48jUebxTqFzJy{tPzU;x)SvQ4Qr1&$FKb;{+jfmMocjS@8E0PjJE-$3hIry+i{IuZ-;%&L76DBm*T|Z9$ zQ!f9mgIQ#cS5UfV#%{VvFQWQup?>AwfFvh#iXEwX7Dr9BqTP198pBDoaz&n4bgxZo zcxMeP$84^-A}ZbrloA&=4?}L=g@8%+Q3U1N-CWhMfkJTW+8ChD;Wu)Ntrj z5{K=t8&OI0y1aZofY3LwBUxLcUob52MN?t|ev8z_-##e1$1H2L&LtsMo%Qr6a4$kWL5byY1?@yPP7m8<*U@ zEp2~BgY>;oX#!y^Q+PNN8|r3?vx8{J*h;>#1+(&S43+o7PmC4;=JgOt^41DCwn%K* z8Y^v*$paj<{esd23MB4uXi#SXa2$t*QFV*Wmn&#U92~TVZSzVIzQv*%G_ zJEXb}Gg+a;ZEna{sz!0|o?PfL3O!baC4w}$3s>P!%HNXet*|A_V^jScvvWnuTBO|2y-@iH(fv`g-CZlR< z2I32SBB~6N;Ki8R6Yt!<$fG|G6SM`tqva(h{^_8ifJ4{DkBOM+Fk3{b-((1t8a#%< z!>g~H0#lxgvgJZgRVpqxsmnisbcU5`$yVZ#6b>aIaWPUhhQ^P{1ed%WbER6Z@=*$m zIjGU1+2S3eHYrJBM4Zd1+f=`z%v)EkrXOKgf-J>#NwJtTnT=vfO$$!US5SuzYPa!H zbm^Wq5((n#w`-YoXop87=lGVFAvv(D(Ey@-G@^}-luWM$6TpLY=h%Xo;S-tJ*&%Ak zeZWu0V+P4I%tWUo7B_$q6%Ri$NAbRa%h%aX)!I?Q+saqrWoqP$2}EQ-rZ1^A;o{O^ z({>uU+7sD6#(^(EOE0mE#Sg)2xTC5i3i7HZsd>xH{5C`N3>v{0^z5&W(+?&~Ze8Bn z=sji6)(;q|k-POy+4+rpQ}GX`uLdOczCd(oR*;I}FP<($BrK`SDV!lwvuAVilC_~^ zif0m5a}w4p-(a)KCA#qkF6#8EC)bLfqarPmF>nS4&~mX9*u6E&{CNP=PdyHJ8zb;! z^EifrhEA)HlbbllA-ZAdJCYFb$&LtAOc22ok>o8o-o|RKxX9V<9ihHpL)H&O->w^k z=5*N`>nS1*e+--na^!=~3x;~n6ZMW*u4IC~A4#IG3tLw6Sl_=zals;&RS2f@?B&z! z)$S9i`6=wxa>F-=MFv~E14{EJFQ zikFKXIz3+v*(l(U9Pi#JIou@5Ml9iu(lpL0K~Ng$$ik@H^OTbyLB@LCeae_DxxM)Y z=fdcV{m8|>v&70WS&c;B6XJ8Sf9yM-;JocH#%`E)HVh zAG~=F%qtc}nsL~%I^m%@^&Zgy8kPCOe}elMD0hO(&)Le8A_kCdF)43({+}ohMO5^@z;?N*6k0Y-U782bU#QMVTZ9+ z^80Fdjc|SmEe*D_{Q7=LASVmd)>u!_QVre(=DU$)oE2M1HnMRSmqhGfjGD?wNsB(Bg3Cr+i>g&Q zNOS9Le6EfmlTykgSt-q+`W?)SH;N;M2s?(>;~pfM2jk|xwv3^Nat2gm^+wqb{<%Xn z@kmYgh)?ojF0H}q!*RbZh^_uP8}9Uo$g+L^E9MWUUJxECuiKPz)T>3I`s#ar&>IVf z+mw2V-}TJj+Egt=v;z(_r>OF|xu9=h+@KWB0l-h9tOu261@22tG?smeC1sKKFDPqS z{LzGFdiHBc8b|pGykdNs+vijrhL&b~@>3BwQ_a!gj}D-RWNVX||e{GoPZ;Pd_-xnh*3_=9Bz!})kS zFO#rNs{Kd=f9J<79k@f+btO1cQ+bx1qPg^3*rGAG?p)V$?LX_Y_7~or8c8>NC~o^- zSQd{ib@W33N4u{9s0|t)ukOxiNAo%z$AKGsM1~*nh2n{QaIR%6GeJWvk$TB5Oi~

FN#!2v~Pfc1*xsil^$DtiXYF}I{xz~^%hFo##$SFeMc z4q=k-T_soTsu91~Hlol%+(QtesbbwxLX)J2uQ6O~C7X2JEh4&2&dO~vaepnLsJKx$ z&C;hMcx&h41-t{>%SaG}t)%X`1}SO3Tjs8x-*GH0??8j?^Im}udv_lP|BsaRaq+PC zKT_IH&0WpUl8tfnl?17}uiI!#pafIl9Sn9ww7|^F?fni^)`#GvAltC-9tuc0i@W|H zD`eV>p!A}BW%M7kcZhjyXpx|sIf808g{s?l{AE>G+sBUs<#i$bLY`*?BMG{uj+xN} zVdJ80P@#zN*ZuYJc>I&k9mzpWyjJb2}F6MACJ#t*+g z+sU6Ra(}fJZ90ooJSYsJda1wupDNVp<^Qchoqxs)OZ3Th0rpNll|Bw_-Y$uiqZm_^ z;tOm8$-}aJBo5uuF$0&YZ2!XheA@S@&*r#aeO?9qZSN>8csB)1Hm8VaOMN|qEtQt;g8{sTHY zupOnBy0DG#Wp$v=lWvM#paDVSUCcj-Jn*t^+n{LGS2X^XFRKs|(W(uVQ8a6n`cEaH zG)~;Ee@QZ<4UO&Kg0=39qR@(pMli9U+7nbi*f&t^N!uOjl3lKRXUKQTO|GqxD5F8D z%C6oXrv8-6mjP`-R$|~_e|W8 z^^HjT6C%KF9SsE!ov)-$qlRU5Y%614HcGZg1U@&D2u=&A8}fA-FKyasNWCCh;8*>$ z7CYTUULpcZ&XJD@AM(XuBm(Q(ZaDo1hn+Nd$#_X8#$5bB0AR&Y(v1SlV+w=mgI$-! zj~kSbm7o_8O-4o=c+{O2Os^!m=kbZ5O>NViq8PPpjYPKtD^X zS|x?ZD2ZkI9t{p*#B64PDa^%&u0};9~DC;x?{ZaSnEe?HmXxH{V|v^_yMW zf|zXCM%&G34_#!FI>eHgClshpG)#VGa6x|aPUtp^_Bz#&&4x0&I5i*CmIWxLeZl1G z{`6$Efe0^IQy7O(i!#xnpIs=yCTu9lVmyLGT=dU?GOoNxEsJh`wdAAN&cOSIL-D4>|r~;1#!CP#X#ysoW$hk_nDouOa zf6y7IRG6v4iBmAj;F0L?_fS5gFLZ1?KdRe5LVgeL*L<7!r&t@2ZLTCY_-NgNKQ^9Y zCR?wVZbtGRLsO&u60so}tP&;B{KY`JmI@&R<%#oL59LTr2L1Ui-; zzF3-J!47;z%#b~y#b#^WnS6<;VR!5Rx?pmqHE$qE{r6?6mZrgBJyg${olxXoCh7*k zVIzD{(awd^2WFU|`!=Blo4mt6;Uedg%^6>64I4UIfw(qaA%MVb=CUe zie5zPAO&@S#C|gVwS|w$Pv!+ z30yiBaPE)Xq-b|@f(wVtU(JP?R(MczqTezV$`eh$$ureE63$mK5~6fL&JE1h)Khni zYoIlC8nj_c16)EF!V4+)sbz;dm>9B+ZgXVdskKHQ%>Q6vEBdFZqhlZbeAA^k(9jaF zW82E^ag;VjQzXjqLw=(?=4zSsTj9VeJ5@d{az5SOXnz*s9cOgDX7n9@1F{_`4Mq$2 zHRLUQ^?df-jyd%MJsf=(9XN^a#E-yl6KAA)>2$Y2l)l`|XeQO{yWcI4lQw(5d;1Z64H2MMfCkL*CsgN;0Uuaa{?B0s9DRna+EHQ-`-Gva44mEVg3&5c|x@di=n(JaDd%< z{3uEYQz8howbpAx_i;&+9QvZuC=aN9zD0Jg*l_~67?hfsro63dV8Gv;l!40zgM~f(o1XS7A5L9n@eS;FG$!DR;LXx$TL=|L zAAOv^w5HY&^Km4lVWaV>Ula~CP(4R>POWseytvX?(1nSqd@kL)d30 zjbJb`Vvwv1k5dG9l4z>iyH(xWH+;h*SMEU1FELg8dwzfnTD0_$6>2GmBXJJYp64G? zSSJzus!rod=)z161V#x65ij1%KR9f+#@>F+!^9R1C?SSqCrmT?nme&tyDDQ9W%+Ce zWKvdCPm7i!!Kw%M&x1d(GpL(mqhc6xB$gwomQAtfEa!s9A>_(qHMN>T;w;o%tQain zt-SF}Od|6Xsa8jPt;3j}>aB>`jEbdyJr>Fao#=Vuz6zu;MPA#;{;^n|ql_&{-RJ#| z;9q^whvl^5V+YxQ^DV7Vua90bU$r+2TKx5`>Aw1Uiu+b=69`C&xxw!k4Q%|wnr{RI zxVmO;0!dI5VJ^P_7>ai9wdj#6lGmF%Fa}MVZmce&f`t>84*ROOg2PYzLOYb)yBVoA zzkqv=%c}{;kaUPcn)Sx1LGDC2sOC>cw^0>D-d9_UMSYL%j**Xx2SQB9d~LDx8acJL zm&NGhDXWTD7ew`8El0?E?UNey<6u#->*klD98D_(qdiplzr0y~Fbn;`pM9|?C`JdS z>-bbrTL_(>030}zb(n#gK+jbYs(V0APUA6NSL~JEt9&EEyROPVixum?G zl)9pREaoX+;g$iGrFt!PBRN$iaMCoxGTyIiicb9QTcl7Go9+H5eE^2@I08d36NZRO zSvNUOw7ddVlDsv{A}2lUF{K|>OEuFS){!xmAG#z1ytXz_csjoY-|KzRlkN;S>onS5 zd-6^10O;fGoH+}OhbbGW2Cb~J=C4|~Jw1$&X*}Kh=kxi;Md0SJu!OJqdFhT={M!7# zf29QWyR2myPmlTy^`wVaSUW+5?LR-J(0^!oOiv( z=XBkOQ~S*P7J+l8~?$>(Od1nWSkDkc}yZ3$`-ZWe@KM`~1G|#XoeSRgKipOy)A0 zv&Q_lxBD@a2@KN`kj-c@5=k1`;!=#Hulw-Q8sVR`1c5f8()dY54mEgVWv-4kky_&jYC5@zJ+jM9;8i7t|&l(0CfAok3&w9Dwx;^_~vuvlh7 z-pY(2%-bsT5))=@+!&@F&YafZyO@^DmT55WcAWK_wJm8&^$Hx?>SSlmo|6f7eLT5%kq$c&9c)?LqA5*(S-n}i z$cgIYQEo(CB#pJCSp9*|PewFyG7Fk2)p_ST*G6VwUM61a_X^C5Ji68d+27Nov>;H1|chZrE^+x*}@v}cQ4c`LlR=ofASagE!nbKy~^pbvwUX;e(};C zyn0Rf6rwztq2{O>M7@sZ9-rusR}xY>SDuSrczlt>&+8MjgcM+z-Pi9af%7+j!FTTx zxPG>0xbm42LpKFXlX>Hxfl9KG!8PCo1j>(^)W2d-6PNBe6Xb$T0)NI{M8Y|3%2!7> zRY%hzo$?BS%_N?D_8+gZLuM5qm|F{0E#<6#e%(O}jAYFv%dF01Y9>FlN2nU{ZHn$? z5Z%=k+86{^J(OcIMKUykQ8fI+{HNU@Udx=Qpxrcd{x)AsMa= zB5#Jr>CKycsEAF{eL}r{N7ID@M=}edf7q=r!|mZ;8UMP}Tq_J}-D9i^P1t<7%b0tS z`{e^*RhBUNnx*wrStrND!Ls-onq$VYlk;u1q}M3I3FWbg#RZgEN$t5S|5Um~d&x<@ zRGF#4U1dyrYa@S-M@UU}C=c$N-NLdCq@_f#Nm$JJe5XH9U2M%;R#gFBXG()4HqW*F zKKxg}8@1)wJ#gVGUl3IFyg!`SMcJ`Dh>nynLd0`#{0>7E5)?BABTRUfdRPeeB+IZL zp%61|KKWv<*i|BC_&U16(Xrx+c||vxQP8R!S~i?WR|`0q{yTZI;U}_>N9MthOqpIA z#EDr0H17WbxN-I;C>K;N8l#~1hQpGM%P&4GJ25eozQ{E#q*_5*{?=q*%;Xz%xt9&Y1T zZv{X>TH?(Y`-IE6J%Bb0JSp7lk=Ls9*q}5^q79G!cX}QApEED`iNM;t;9|ayeul8& zq?~<4^_rnsLZ%M!IYDenU=$v;EA5Qj3RE5p?(5U>OL7|IRO>8x>CrmEC;i>+_&vLR zM%5p@Z?lt2np@zDjU3|~<+UEeJ@7qJyQsDJTJGRZ?4K~pFgPu7+#}Qwd%2o*+uVzC z_6{PG<30A6dcn_5G(1z-!SR==XBc5V(_KT8+2+>*kn;2OSK zctR?Ks8OQ&Ic=ur>uWtLgVa&QWJ$qJ7j@>5{y@dc0lvE;V@T-)>AQ9iXSh?}ou6~I zr))?ZS%GXa!`ZG&Vw~#)YN497yh#|gjs^?M*59qAvsa2`^s7x@TZhCxcIX1;DbA>e z|KOfx&&6+#jW^}HF%1g0d`<>v>-V>zqL2>_yK(JFqaqERX`JiY#3!W0kB}M3%^u>_ zTRqdA4m_F`fah+WQC0@S(5G35_XLN;=hJ_;yEJ_*h`cdlo#VSk{J9on2q)<|#E^`C zaDisj#Wk7#A!MJ1)SwX_Cw;F3R_1L;ptTsdh^j#8Uyu(o(c%?dG~qF0T*rvqS`JFV zeW(k}?IMcC&wQnRtPW!0$pN99D^Ji}4`N&w0sN=dx_BYY8UujOm)pDcqOU9)-1l_T zlP*zWy_)Xdf0tv-0~AeAL0{k<)@}rhT9?rc%e>lR;aO$4)N7X~;%_{A)M|{UeKI4p zG4)2S$OR*fcS>_|oowsJpDM2%AGKb5lOH;wbi_1jUxSu?_khQ)L)eJwncRPRVW;#BSdZW1(we_@QZwNP z?i=fEs9^~$POhNE8O%c~ZB9R{g7VOYg-O3ePV_k}mM2#6UHmL2_17mpsO9j0-;Nr3 zD(PYuS`;_`X#F_1gQjzUBFF83HJt8hg9Z$wJMKlA+zybOn-4wq*yakYDGq!I2l6X? zgTa0QBE==W_QbtW4=e*M$!FdyD;-#hqDB7Xli*xo-m1OmlX~s!eu|8vbw!>AOyfH7 zoCN%y@xfsUFugF2Lp#qg>|RjUzOGFORL}z6u>I4qG-MDv6iK=D!V!oho`4)BWD@4J zjFOyQRWcAO`no#>sA%cf@ z4un=|;0)B{$Q>l-lt8nvl?Fewe*w}X16n^3%CkMD=Y4~TSjo2jnf6};4qke9{&;!o zCT4WQmW+|hadJKWEebaIO-Tc!yhmo*TB!lbYbI_Y%10tNRf|*mdHtPgy_ z+N!3%cks?GRJdnNyZ8DPkn2{U;xn&}_+w5XoNf!c zYs;M@c5I=mSpd=B-qW_-#PY3r`%f1^e6V0=`uo{~iuQWP854jrZpr>_+8_P}I5#&Z z9R!T-anA;FlW74-_wB*j*lkOzK$rTj+EZ~m2{HRG=cac2T8`Tt!GioESOuaDfx?># zb?G>_#jbIJN<_f>XYr)!=e4o3r|{zZwOIF?5sGHmiS|gmB=afEf!t&oRSro4_x0bk z0^C3}Sei4FJf{<{Z+q82SW0GpR3z!+ctsT1=W|b)a*$!V9a2ZqQ+)+~P!jDXL@#2& z&g2HR2G2uE6+)^^%xn+h2h1p2ozBl=;kS5Bw%2e>dGUB}JRhUOimI znVWH;YG76V6u7>aJy@S^eYD77AN)X0UHdhH7afXN$HtsiM}0IfHYu5lJnR`Vr`T*6 z@lBrA$!2L??)qvX^T`13VF8BLMbE|av5&yZXE$sHSaNTk&44UfD44=kBpKp5dkn(pY zv5|cZLhlBz;Kfj7`1#6UjIeH3JJlVs?K`M-wx=^^kN2iNvTVxN2Gbn8l&J}V#hs-r zb!E=$604gQAfTLXBpC+)O3l!;_Q(fCEhp=#wK@<(oc@$f3qI5#@IIBbJ4wMHFuBeu zY1w!0K$l#h5$Oh$8HbqTFhJlqn_Y(+q#;EmC}W2IX5-L-d83WoSo_8dC(lIc*Jslk zlq$<$)Te73IC^1WlC-FAPZd0V(v*rZF8SmT%{m|IExcdXX-x}#y5b{2>z>#|X=ddg ze*&HHE#dp@t0Kz@l7ctLnMe+ebgzuGAoTSvfH@hu_Hml}t_<&YH2QnMMn+PB0gd&i zV@+gnN&+i7XQ6c}#_Q=&5ZUMYVl&gXYc4GKjeK`g=?m5;{R_pcx|hMc@2}e=RQo+~ zVP&60zk%B+pJl1Qj60xIlZg)J&R7Nbptz>h_d1RX|DnyR4EpWV^e^6*VXyCgng2iwWPVE=`VM&ULeT^iwz~$bPTn^j0{3BG{}wMP zVV8pq zqx<=Nz^ln)oute&&)K<|OADD3k3RYK=eU3Eq$@B*ihM1+fAaC118+jxfip=#dt08F zAT`%d5*MH!Bw8|{B=&Xi=FEGJ3Ks=|!Uya6HF6y=ElO8ZZpEP{XRtDFve&C-F9O$q z*qWNPWwlp{K;=_(w{3=~VVT~ji%_D**Or@?{P!+#-hytAZZuswadqEOytn5~xB@ZH zr?jd`@sq93W-<@nBY?h%gNj8=t7Igx0 zXRQRZ5tN*lXzn8|SmoCh-TOz2cE(vx&*)_unN(j*c=QEriI-6w-ZX7xY~-bF7gYY| zP2fe$eIM@)zYkDQzcz#c*4{@!E3SIC1ut;ah&r5eEOmKovH z;GnL}B!WEF^+cAY^uV1qI7AewAUYAc%PD3d&f_O~L%O{U(KDXAyzYt4pmmlqf)EUAOpcTy4j|_scf0NAd~X$d9&3A{-){AM zWB=vaDPc2j>l+Lj>UH~Ibmb!evpm5t^NDH_8P2rij>%~+8{>LaXHu074^VlzSkM8<^|o8 z@|3xVfrt81DW!{&==J+Y3y zcca^Ra4ld;H#KZSs<1>Bj_*Ak8=D&=2E!-~I^bTT`Z4>t^qi&{a+6mVzMAC3!Oi}m zh9cMfw@mw!Iq?OQ*?imHIbiSt+5$ai&p=95Zu9FQ?GeB$d(dHnFVUKPI7OOpMLDf7 z=D|_so>uP0;)jwjW5KcjP_Yb+z8;A3HuesHv6Qm=nu!%)1lA4!Nf>W6>0z1~V9qF! zZP~+`X6%Kg#(|UPh6vAqvVr0ckU7(Qn+N*yzZ#xNu4~K!OoN2pDR$pGVwQ^;SE>&B z6?C^!zG;C2h52Rw0sUMz-vIyqTJW{Lt$_g*>sHmJUyAgX-?3cRIJj#t%#+CEoIyxO z-W5tW=D*A-A2pC27&1pmqA-~6z7TxCVPs(Jk5;$d2?!0-=&;ZJ`DNUTVBtPtoRDxK z28g+uz6WWOSDJ3scsh$o6^(aZB19sHLIf0Pm>6%gtP@@k^V%CZ<62utsmhNr+Jun* zx>Ts(L=%{k^wIOS#WwkXMPJ#kTbmG0D&FNGr-{0UpiDB4s`1Sg#117T~9LFwU(%CNrtfoHmuQ+s$+__u zkE@Ovgi8+H#%wDSHPGu41io-RowZDGAEoXaAGi#)O5MK9!cMl^I6{?R2C&c?3>rhR zqr&U|h;mfkNjxNc-TaO)fYR+N4RZpMFeBNUbQ}3sNbMKt)sq%!3VR6phtqC{LC%zN zOn;WErHUY1y1L=0ao%0OE^jV*{SKe3%p=U+Bhq5sCRY2%37MYij_R@hEH4<*W%818 zOBc&UVM}r($GIY>;b-I58b-6?%gt~rC@dgu=Zbe^zQu&K?+!oPeW?nJXLvzJ zaqbVYgu8X@ATvslMm?e8m&|QYKEd6Jm22^Q=%?K68gSv0(4K}s<%V~CD_Sw?Hu~Tu zsa=e}n_c|PrVSSTZO{t!2P->PWbHS790Xc~F3#}3Mu?zE!9_A?h$G0I42cnz>V671 zG(R$L;bPtK)VZ;3r)>4t>C*dkDALS4mZL4?CFdU9<1NAI8iB;5^_bxkKg2SLde+=fx48oIm*EL6#T6QY>&?=sBddVmgrk}c$VI?~Bf(OLjq2J_ve zNLD>mAhiQhrC@$&(W~ET$l26Q(ORetx?WLp!MFvy54-L>pPCVnR|t>dP8c~pyv*)n zL^8LOf#lz4nLT3@8hf+I1H3%NpyiI3mycL)0=Kc}897d8vN)l{GGcTFJ7px z6O$dDXAse?yV%JVJE@H+0KPG*ruqBVBGZ^jT4gaW1&0DH6uxu^Y5S7;=UHWU?Iod& z3OMZS3i=(a?`!+YrTcSkU|u5gk!Z}~5N~T8jCEfEy`%~$1_1`EFI%rJie4LQy$Cm( zl8He_VH3>ANI|)Kk_`cgW)`LG#V>|=9MqxmM?{|U<>2Ytxhj9aa^ABxwS5x;&LoZ6 zF+q`d_~h}RCi$Y1(F|?}8<{%=8_qE~?KEdw#vw=nfP6u87PX;sgCI=yfhx_U`ghQN zDB-5qz4@`8qPlrg==u=LAdi+ud9)OvSOm?4OIRqaP!1#1yN*6ENW+Y~K3!=)P1&8H zZnwWMS&7_p#B`}?S!S;B)Skqb^e3%ei~8P+YqnF(BMq%F!!2FOd&<1Ds9xh^9;c|j zci;mwB%~exQITB>FA)kN>q-HpUwsAmHyr0)YkjrQPel3Sd5zTRi(h~U@$ElP!E*oA z5bM)bB~kON3zJKn{MSKMVNCgB%-7q4(Z7Xn@38e-a3)SN%m*FdeL~D~9cw?EC-L#7 zI{+OfN23f-#dW6$ThcFkt%1y zFfi{LFe(a6aEHcIq;JY$-`hr#$|_-G61ONTO$>9TeIP=Mxy7-Ua7S}AKQ%uUwss-F zz$vqN1_=VEN1`VbI{Bet;wG7f6x_knorqUg4&ZPHi2|o*9E9B&APe|7F9@g6%2ti> zaBEK?HT1uH5QfpFrXNy2F-L(#MSarqUmmWD!vKeaieq3ZTD-ns;&^Cbakj6HZRFz! z=7cvTbtn5>7n?~9wmw+bXu$Nm12Xk$IPsU}lhiPoVX<&|cFvjKB2J2bVhHiO1$g|{ z0PUUE(}b_lTL*N)Ouk}q5u|+2PlzbLh5#0j{jrv_#DysMfRD5{3{YiLKnitofKzpM z>x(8aD5l1O0hgUA=!FC6m8uy%Q30bD#&D>WI4-#9ap@c;ZUNAP@@kJP`*$c<{f5Xf ze_@CGDX2%RiXvgu>l;|+GG1KJT90Pvv_@SkdAqz4+IA-b;S{&)(W)Ua{uQZ>$hW`M zozwoT*dU${XcAM$(Hw>gRZT%K0C)(b{~P|Tf}_=u&U-*SXFfQFvYpu#89r2Va=yah zJZ6J8VDoc)z`V@%+zX>S)spCwIp>u5opv0^&LevOz|GBWeg$yd3m|V8w!Mx!fMo&y(Yj(vxFm@xYP3UNR9^{UHY&-if5fC^ErR?o1 z%QoooI0ONb(r@a#@CXs_vYeH1Hh(&2y6V>MA1(vjPWZuTai~updS!#)7$zH`jhY7#F}kfp+7yw`zO&?h{re9_khwyqlhtxtdqQW zGI4Ic6EE69V~V%$m-Gw?Px`+_QvTh*dqK5F7--L;-;jSRN^P((&>!5X{CghS+ioo} zIj)Bjx|f#1>-o+8cQ+?fyH!|*5{cAsivQSJ)v{NsS(F)IE0rAa$}*5vl&%Shm~>)+?4B`q zGx|U+cAQ{|6d+alFY-fY_F0eSeCH9NT?(&&51vucmtmL=L(;A|s(-m%U%8MV#~T*P zYo{8KRu1eF<3qF#h3{LAA0DPYwo8Sj5Hrq4n(}Acqt}Cu&fEPkH5RW1*j3YPEA;Zy z#D!HAdP6eF5NMG7m;z=B5vD>GFogMp5Fb;gi-};-#%w4xv9*Wq!P@5|l)f^4s>XLpcj_ z_1l;E@Ho@=PJyNRFoZrlvV;6c11LBXLxiG^|J_>cK|BI@33a_N= z+D2pBwylnB+w7Pf+qRuf$F{AG-LY-kTzkFGcko~Pbe+sqvu2H|aci{IUuQ^+4!Ufj zu%W3ds2^J?p58Y=Nr}|OERrLrz=kgGdfRk4$&7Qm1Xa#dY}36|tgw~Vq1M_p`Df`) zc+A>Nb5f1h*Cx-}oO5aG+*Pa$n{B7niFYTJ->7%Fu*o#<+#K<4TMY5bPHg^Uk7 zs@@Bt-KY!o5x&S4G@w3hB&(KAZJ9^8%#fU9TQXz|=w{u_qk=(=U~2@&&4zI$a0K%k zzBi8y!@(>k#_4Xou6>3xCV!Oey9}2a&~SP9D#Z=p=ZlnT+p55e{d^bSZM{y zAO(e1^wQl{kPPa(GBUrYj(!@you+!T#D5Qql@4Gx^Wk3L9+1$Fl{rO&o^rq%~ z8gCESJE_!E$r-VNOm<*pGh?2aG;-B*bVmJYg0Jg(+ZJIh;3WHE!4cl4Xg|A_RyZON zh%*m=mi*Q#%J^5W<+rMdl;w?TUQ_PSk*z%9L6k5$M#>sa3i zUzCh{uWA)c_liJO_9&{Rw!Xu=iWAnpWJ4)ad(C;>e(Fq*-?2YRKhFtp50n64Qjh7V zv~AA*gE`J$phCe#N9p$u4o%QHGIpK2 zlPjL@>5P_kuoowmHVy8vtRf{iv!muHlr>)TT_pkC57Tvl?O_!uoF|z^_K9*7Hr~w~ zay7+Y@wpTCLO(^?Opx*mUvE+lip)M=cFU)MQ>)AL&!LCyke~dhBQk=EDz_^383=9h zwb)WWOWfQd;=!CqPKeGS86suZX4wk=H3>p8cvdmzttDW9^4)g2=6k2Si*o(!w@^iw zK~@93CNEYSio5fCT)#JLEc^I>7#!{fMzPTiYRRg^axptcpVVXU6u30%=_fVLBNTOy zf7{OZ07ViN1Q3CPxdb9`tz?NbA%ZT4a)lg6u}mmBx|B34nTizZvEp7S>f+HIO3YXY z;~AwEnWC*-@sjN!`z{9GMpV$!r~n~BnPULAfc<|MnR=Fch%+tM#uyhpRIaEE+S&|_ zK$JY)@0|SNJq$`jD%v$ns@@T`2!K`(W+xEYW+R|@1>7Fmt=W!B~hD zHeIO^bbR^SvDS3pA|lU0m`m{Nwh$OQrpO9Od|>FOtfUe^Y;A@d`7bHPnTylk3{aDv zo*^DD>~3ZMfg9u7fo_q9K<1sFc)c(*s9=yCd-!hUgx{{?ApnpHAOAO|N(zfSr$ZOF zbKgA?erzLC>c}dEj0$w=yZx;riA9aO{rj!#Q%Hl?lS{7f=A}E_c5&DEj?Oo#$8nH9 zk;&)H@`i?(@#Nr%LC}X@0BQZLGF?gf6t-08>VJS#Ov9Ufa? zt#UVoVq=vWXc@My@5w50(3qw1{|}JB$Wmyv=2xtR*%LBGZlFA{vbT86R>q*445;za zm;;471l;Y?cmcmPeHSNyw(VglRGzcu0eM~BJV5Th<%w6Y3Jq1(9L>UYJnI&OVyJmr z^B_89sELCz1(_QrO!vJRGOs^6eakQfGv~thTO~xK7ZSs$jUqo-Jcxp>6J|qt(GitD z4c1g-#dPrtY^}L^lF7K@et)?&V?Qjp)yu*Eah+_jWSU6>Im)(+s{S|u!jLAvF(znF z^>HfChh+B2!%|?~V|{cw`e&;D+~30~?e#5onH6Wd7B4$$epn54eFe3jCMy$#x4gbf ztkwR$=czT1Lib}rhQ#oVbFUUfGzzt!F!jqZ)^@=V1d=Mk1VB z0CVUkhJ{RY+~!JM4xwgP-(*?sH*(rh%OYp~>8ItYQV!rn570NwzjPTe94oAM=+Zbq z&4qZ3^0TLBu;_?BVAEiTm+2{LuJ+WpkL<#Ncc=6U#E^21s?3wfx_|KI#>$*JxAW5| z!yyS@B(V`_DcPo22I2R#dlJK~X5anV8RtU=m_;YfBkJhz%mwRD^-Jcd$#> zk17O))>X-*=D0YH#7C1HeR7qeUO&;sDr3~^67@q@^BNn+=h3H&cg3$NHZehb-{Xm0b# zXTVIG)MkC%7KhJ&fq66>pBv|;O!Tt#%yN_{XTwyjW;OkkY))=u5G>H674`a!nuy-H z8=k)P&NZhU*NPvPreyvFAX14ZJcS&>>{2`yM#i)QhkFDR?dhljgamkPZvY!(re^?V z_&+sMu2R@pMBB)`AM^_xxHcUW*Iwd`xOOPH0(j>bJ-Iw-dT0LXr(sUla6+k?V`{vfU-2;`w0Hc6hG9JJWJ`iy zPU@u8GITzo4K~W5#g$oM1@EfGs9Iq=+StdpO^+Ro?-)fIV!(v+%Gd~zbd+}4x!k*`V2}4Z3G>v`#u*XcIP^{K%TRsean_k@ccB-18|Ay z!@nnh!XQ2Y{Nnongm9p`t}Tjs@LO-*G@xI-8pZYy>J&kNsgT#2=7IbR4kJ(Is8WSC zC~=FLJ4clxjKGiL-5>5vaI@>govC{DNPJHKj8GA~9R0+USCh-lz`xXu-_No)hdi4W z-GCc{=4;&+rX1w8bZ`iPH!;b?^is_h7dJpU>{cj#dkV8)D4XgL6 zh$@!V4GyhC-s-vYLK^%Hsbs%o`w)}lbVvN#UJRR^OMNAmrMSo zf((m%Ul%?VTf3&|C9COD$}WKqzd^$46|-aw749HG`CPMzK@_wcaDloD@{yzsQ0Akj z2!YwTCB8i))Z~vS9~}9hV3Aef5LO?FXJ@VKFoZKaJ>bBphQoPF1(0X{j6cgQqg>2I zaiK)K%0m$vfvx9DYXFUHz{N!Y-X;L~LyCI*wXZ4V`4h-lYrX)6_jOaU*8p6uf=#qp zb@h3Jf6I;L1zJ(fyB}xI{LJ~B)1(wHqIUMjYUX?Oz&~Cd)zbh>W=*U?i#7RR7~_C` z7Fs|;Lc%}YH=tBfi>29r{S0VpGsR6*F3z2T4i~;cRO-}?! zg)A&^0%jz08Q$^|cS=fHY0GK%Lr5YA9!0RJ&h(-?YWsP`Jj%wxy7fk4_#43mcc61{ z%stGcF9tmRo+ZWG_ZZ4ky|SOW@~v5FjL1eC915?;r2jjw=EvpYOND4zH)22f!wWhi z$3ar1!-}tQ9xH4)J0AU2j^%ty;QO_{G@TYH#kqg$MfRzHx{|@XYp!JBa`APU#Cj?} zwBy9-{wikK=V5;xH;HJn{k;EcS0@B;Q5UhN(LVP%+7>498Nki0a1Z#_!3T6DmC1bf zL1H=3Eb99Z&E?Ce8j+n?>m1{q)G+cH3x}oKmdI{!^Aa`=A${C%x-hiCuk)5H(Im*`=wX2X%n zXZ%Y?SYy#~eKy^v?cS+T>dnT#MKhoU1S(#AhkDoTlDlIFOxC+~{gYucC~Z4#2w`N` z?FL@cGB?PQpZ^3;mje^4v!zLL6TZWuYRaN8^xfXcmQ(&u`@ZJg6Fn&Vq-6P;jN)+k zs38T{{YSIzp=^Ahcvva(PoR6tj$7HMHb7*A_w3@k_lLL!uX+uA_Jz1~kVR525_^n< z)w-?S3T8)a+~-!6W{!`qo8p32RjHqm68L-G;f3uL0Q&4JpbGHo_|1JdoMKYb(QXNe z587NVNEFc5uQLxgs6GN@dmSo{n?Vm-uE#?F;fG1(fCUne*kCXI&+-hv zR+Yj;OodZhm#M z8ES6tR_6hRrMZLur{D)8W@dT;fr-p^LbQn1k_%8dCyzfIvp@{7(gMR6ymi7AU#Yd8 zx=(@NYWY^6JFQ0i36x}Bo^pGAF9_LR0a>Zb4`ROot>fEKchk6nZQ4kEnKvwAY#lwAkP1AI5av$BF|misjX#+prR1bVmAQlcOZ#?x%| z&}=tHvLP%l!tNZF-0{xhB-el9kBoNy?ww*V z!&^{wt__uE&COUniQE=bngn}jQm&-DIME%DH6^=Yd5;TEoVuAW&xBjcj72Su^r!F` z1Hk%NK_K0b3KYWwl*^igVsD~298bB;_8!ET=xRB&(yy)Ig3Y-OgRmtJmO?OVGQLhr zFV3MRmRr}mf3<+c&2qRYXCF>f1nJD~e?GD%Gp?)vrr!MTU>$?LJ$yOjhXKeNp9$Gs zM5BPd@Q?2lZixj?OQOsgK{{J`GYOmE&vAh(7Sly!=;ffBzqw)^P(gm-e;_4RJh)%^ z3OySMKnnscW(q4!Vx9w=Ed!L~1lMd@vPuTfFH@7V%RWXJHYxvpT2tto7!RXZa6ZD> zQNcFfeTY=u%!!Vd|9IQPH!*jIW}UOpWR)@M$uE%D0K0e(quo#AMF5e0d6T;A0MV$zj zRFwB0WJJsspiJywAUF_#+nwU~XJQ?aM;0&-N04_s?^8v~X9Gf(#E}6KFbP+Qk7YYJ zh2$)!FCPUpIxOh5e+(pd!8=#kUQd(eDQ>-N$K8|7-9Ns#2dHzsAcB~8T zoIZJo&0(-BX3$|#VnG&hiiqCwZC@X!{|9oI+wA;DtOm?$a2pJ&K`6TK8`q?pGA6{nE8-7RqLd-hAd zV@qZ^w-`ZjE}qOoXfBaJ0m*Yd)(UU46J{ooPg`c$y$nNvcpG^n5*YX)w{K`0{O`77 zQS_2hsU`R)E!@EyIJMPDLk znNg@!Y+8!>zps+dUKlVXJgaQ7_S!sf1;p;l$4?r0T3$r znxgQ(DiB*neh##}R3VHd{Bhxjxt=jzhp%sg(cp$`KB7l&bzR>;ZQwlBE*W5_e@$in+sA$WJ;3g; zqV;=HSEQ}=`@*yLn5;=`%51g5CbC9=uIjSc7KG7w!~p&%S<|p7M>fl=4;0_Qtg8gR zuj(ioGhch$ZbQvtgW@2OWnOaQ(gZf;7rNxen4N9UdGH|XGx8e4bM3{D|99CFd?0K= zAn0Yt2{ZMoM?dL)JnC@%B+w<7@&y240&TDXA+eMcKC#*R09L4z>Xi$(Z*rB9AOe11 zz8On6>mLxusD#UT8!dpLuB22$vIX6Nx;>G#b8o0GXS8vzrC19B@>)JY(cv0n-K6k( z%|VLy`FX%Rx4tSU123?}RLBLo!@{q=9mth_eANy0OPDgb*QjRJVAL*xL+5NZap8{7d5Xx&Xs2HpnmL&!2naQ)D^RDn8kJcWjmfR2g zZCMz!N)Liee&}i*U5$9yHgQ6wzu-LQ6EE`R(JKN9qk%1$hog`9Ru#WY?N~?U_CNOG z)4G{;z52Wcw$rg?`Bt*?E;P2e=fU^peLW`$PRF5$$Bn3uBOZF*&q5_zv2}9!Xra?f zUjX!1LcUOb1Dz(yr=&nQSG`>W_M65Hpm+c2;K^zA03;k`>ivEkXqgQ7I1khf6hbK|6M(PL$B6T);hu0Sm8utP0GrZVBibQ?h?w^>Qx zrDt`2#zeEG4s&&mWOH;z7h6pErGt)UP9SHNk8mSG44N%cokTPCDiWF-?P28wlDi|O zG^<6sp1SJ)MpiZ9!#1=7_BZdRuXn8P=)dO&oG1?WN7@-c%Z#zaHXs1=0g!TFJP)v4 zbpU=c9*;*J#nd#3^XGj{qlC7wCeD)v`aV11=6AQ)o`)TMBXoG4Y;9V31`(F%7vR*6=I?-RVP*EGl(ldN-99YAlfA6N@nQO@M=Nyyn9 z;X06ylpFBIfBhg=DUM2-`7|$B`gL~VQ@(4PZqw_s4U94n&fRw0iX< z7imq~W$biAfIo5rHZ0wTLT#`fl6v%$mI#9X_jCm8 zqRCAqX$B}0AD^)yj@`BaKP!1TolT#v$Hr6p^7#Fy(YM~(>Ki9IB`sM#Hq}rU}^_-te z5E)5tt77Y6*$%-IYfc52f=&+?TWrzgL5kwF%L>`!FN%92?pNZs+wPps2_nu4|e z9Ali8R(G0?{68i&!&0nN^3)$q^c`}G+yR14IQa)!AkF^R$;g;$eA_u5tF(sps=yXI*ZmonW~FNfk9$%RVNYMK97?+M9+8ZEnXhomahF|!c^XWAC;Yonsn!k1o`-dYdq zf{F-kQ;^@^Z3l#gJ~D?|8w=7urXI8&A$m6D_&CO=q!WNX$ig`T%LXl3^_8GSR-7ea z=`C6j9Rli$U*?**q3aOr37{G}G2YT;x0>-jxMLeyTbl(X1UnXZ7qoqU{~5YYEECvW zDX&*_i#Psknh!AzxS|tsKFV!_uB5P`X%*1Sd=7xa3$VSGRPBlEtCG!ujZ~N|h3E*T zs<~ORW+nFY!_xGcGQj3mQm#wkgK!w+^mEL-GgFW}c9Xo~lta z*GF56pTgo!0;^cJqBQ1YwTMN>`;GIIFyU#9qp&Mx=6KW++#W1umRj|XHX6TbhE?`& z?KIDSf2^q2{i#|2Gab!~SrQs@X{>%VuYrRN)M@9U2))NL&QM)qGF&4Rzrb81_k-a7 z%+flF3#?&jhD8B!=X4oZ&C#8;6*?SPr+FORZV}4EfCOo|C zhfMsJN;?XEXubX)xIFM3O~?mGJg|{crIURH=3XBX1YrrNFdG?iXb!&Ar9>UXJ|8^! zkqu+UYy1Yc64r%TQU}n15RsxC&^{-`fj&`JFutnXl`cm+eHNn4F^2;c(-i&*Hx|HH zVRCdsjulV|ba7kx*68)Lmr3#Mlp#cFw7Cue+yp6n`tqVkL81^*P6hH2`*zL$;VMAw zmm86#mc1we?tCW_keh_O!!2W3-#+rMrK`N(pxuDaX(5p(uizlG$m!H^Xque_pB-Y6 zgyq2$bPxV#18y#21IF6$%8WKPyrUt(0@{F@6KO)`L42niy=dFOX**aI@f+z(3nuLD zYe{nIua!ypWpM$gDpi~n;S@)+#tB{MeHtsP^;r>}-N;;xA6wr2cR-7(&{)|R;DaFz zUKZexduA{!c;?g=|2n;R`DxIv69%ByN*{R;B6vwidCQCV z+LM_C08Mnm07{X|KoD*6FGR8JA%=MUX-($1mr8A=#km0LrhojwEA2WN<_9`6X*m>*j;LMPora<^8|OgF}O^#H$=J(PlwB)YayV zpY?CZB{*Ok7s-VejYU8cgx*+IT0e2hn|SZ*E$f-!%pd!9nOE8Zcx(#UJOSB5+abW{ zps_a_A1mEDV|CgGU*8k5k|%u9=nYp!#sljMp~XA?D!f^c&F*1cl1pE)GC)(@%>26! zo^uQ^^G^}T)f~11aW2p06JRV^f(7S89%xxxtw>r;T*luh}t6RDH>xYMvs>R(X-h zW!Pm3ZBZ^l+ceRI^k(bb8rdgHSRrWz2T`nl8(xJ#991@rM825M_(P6RU}@e6hOK1i z|5{?^YapSj`5W+OvWv|LR~nVT(rLDF!kI$Q77~vW-Uke*T=x%MG)f-{SH%&&m{?}B zCFDJ$Ky#*4g3+3=l<~Y)FeI~8=kC8fnXvWeiJY)x`%z{{@sk`l#&ZuNU8rBSSGtE$ zM2voOG^>^h#>$p}GeJ1SP52LC_Z7S7z*VGr+$1za>tS{Z4^LU8BLcY};gyoehV1Me zvHS@Z>dDwx%x|c!J&*s{1=tomVlm)EmA_N=#yKb>cMa{iGVM9f=hPV6d2v|qVD_OC$u0W@fQyu% znDU)x33RM|H$|OfQN7YaGYcGsWDt>h#DKVJZvEJRO&F+)jNr{D|8DC;exJAals|&| zlNfdrJZHOW5}`+ovwv(#J;MUg^Aid>;v}8DXy1m=Ex}O2kyZGIh(4MY^He7Y4Q`RQ z+moYyI8@LBP_5_3&ym^!3e(!6nNVB8YF$$ihmULv){vZDN@{iW0tV$6D=@`s((^Fs zvEDgeo$Lu_k@$tks1U=<({WrVFuSpG^nC-N+CcV)=`vk`^S#<=UF8^v8D#t`y~iU( zWDlQ{dz`6{ywpWNM~{Z%)jF<=sp3Kqj{Zx)PBj9jFMEJHs|s?93x-sH(GWOBUn+sQ z3&d02dsWrceGk~D7DsarF~YW*{Oz*&{6Ca<-iYh}o`OtI0=}(<&H#Mee73+)n0x;4 z&l$nr)4j6m7|*cK%r0Jb{~7l_E6)ZODHn3SQ1&5AkX;4;CAJ>p@F6+Enlv`xc1tCv zl>7|@UYD_AR^nSn=J?QE#!3dl&m9xn@Ds>XZWvhq)r%pXwdlS!0RZkrUxfZ$S+tr} zVO-&pkn~6ZqM~BF?pFXgGG%DUit63!#s^0~kx9h=kuT;ELXY1+hX8&`zt|mczOljT z`dL1wGjcsil}cK-(w-~s%xVC*_s7~6WmcteqHlSzz|mm{&@}*{FxYYz%{jh&283F8 zn%6UvlsFc9HD$sNB!`b%qPpz`!52eyX7drl&6V0DArPV&HW z*T2JD8x%vkaiPE}lnS7GeDOXAm=k-e^J2{=iVm1ka=#>=w7#Y;H!^AUPn6{~*{ zb3nLx_CLMzs(%>-^t~W_5~|8$A$0pZq+dOO$$kqX?OYgR_ls%7Cks92f;PA!m1zEb z_Wv4?zfc>!-6q5xIpGuH_`6r-MKB7A9Wnv z&oiANeU}10&j^2F)_n|~jBdwmt~bL#r5;~0fYxA6Uj9EOf_kwfsH3``5r4>Bxg$h( zgv=WGB9<#d;3fnE5rlUOL0}C=bm&6m+)~j&1QPfIA;r|$6d!p=5I`;axw9@F5N&T` z*LJ{Y+ZKAsdx70M8Fp;Oa;ZrjeC_O?O>#%uJ$q*YLSCZaZ?aR~MUAAPw)rsTE<1H9 zahfUz<53Oj24t$+TxDUZ-2}*kpZ%#k&@w==XT_s^@_~h3I8*I*y&-pit|kz?*iGLq zfZunK+ zhVF`J)Y>mOKkn^U12K#$t9HHq=c32saneu#wDk;D?9^}-<4J>dJ|ZDLl=^1=c`$aK zkGLqHAz6LxNsHlVmGz#>qjO4HL;*TQ*BD89|It!qB#S4JBGn&4wawRi)2G)Po(q)F zjng@M&4vx?5V6k=VKjnOpy^GLjU`2Y~b|oe~~OP_*`NaK4gUeN4O!`O2^?d--5wR-kY;M%QweQjeRR zlnz2u1u`m4SlSTD*zJVyK>JpVR(Fx#_L#<}Hz5Cgz`|^=V;-d>*4U;ss=EP&g#iNm zcvmESp3_^!g=u|0iBn>$yFD4sby=CNov2p~>sr7ujNcX6ueLlY2wEI9kcj*zYJ2kU z;yrq@y{fhohwZ+r>8;4U9Yxwd!o<%*`n-wHWWQQRQHs}LHA!6QGvZq+?J@VH#LCi< zvjX+Z1uiB3+CvtshoUJvCs2Z`npUM0!^$8o!Y=sWurxI%I$t8q^}v$S%hFyJUAR=JUUenYs@DMMqgb&zME>?bEjo;{1vjwVLJ#I$fq(cF1hL`kO+AOQPf5G&s;!xpwtk zT2nF53YH)ooi=3Lb#m^#=ZCWtoZIvs!_fa+U43zy7Yl{&Hiw1{RcA-6)J(YyK~YnD z&A6na7K7FgJ>Wk>Uy>PU>ahcBs|%ilsYJr~xl&3C>(+^>m)jj5qigY*K=BWTVf^Qx|JK>kG#{5eZIt0OrlOTiF{GDOv1(UhFD6e` zLm%7KxwPDm{}uN1%71&fx=~Z9(xGaOSw@`_<>?@e^AV`5)ssca4ATZKyAcp5m)5TM zmZN;w6-lOrfQP}`sH|K*8^NHks3|5Es`V@1jL7~l!n?kLJW^AqIv;t~k{g-7!dcYu zO5{Yb2v1jtrKyq)WVlz2PA7cX&X02g2~F9Tx$>e)h}C45NQ@Mn)(Jb_3L1=9;8Y=p zBJ7p4(cmx3taxG;$QM{Q|xL)|1Q9 zTqY?cIIAUEiA*bOq2cKm9Z6GQod%Mbr}lC-RUkuIt92?43-HFV5k8-hZgDANDprHa zb7f?+#;q-48u({Dqr4Y#NuROhF*lJtrtj`%bB@G!jOl1}ll!l#zN@|&Ju6=L+Nhvz zH0|BhsV)CNw>iH==2!7zC4JVoWA$zcfgrPAP7UM?4je)vXQK$awx?`<|TZQa*AG3^&GzQmyfomvk?=YAJWu?X=AMm{A=Re z_Cd~2wkrR?@#+eHhw9i64m1d-mFDz40_JB9vX$S-gJV2acGrUh)!fr@i<$0|@ZVo! zGhhwhcg(7(C;~TuaWA<4Tve%CX)DTQe>tS-l1lV8#K=wlO%>(B;DTcf3DJZQK;HBh z{tTMsJBoI-m2_s5_rBLZ>QS~>lmu|eh10#!+<+yQhE$V7n3u~CV{YpXsF_5`C}o`U zM9Owt#RvMk*_J>gfiM?)CRI7r;qPNpFSj=qT8ZzyWr+0Ep?plP87XQB2f9372oz#; z(X^3!V^*p$H_-SVl8lfT;B6Sa6P!cB7FmbAsD4nqh8mgma?dEwT-W3C($R`7Gy18s-94$G&*nxd|XvXklybgW3 zPyf-I0jox)pJwsOg-MF@o);)Ud}YR7%3`$8YQi@H^tuup zO+_?86ts<$E8fA@r zT_P(=Bw+)uWOQ6+@gOT2`1tEU2YT?lw3Jt|%bQ0eN-Zs|<*)V?%?PKnTW&(ATE)&GY?~gjpjcxi2}pELw)KxQ*3ae+T8Xiso7!79B!LGj zof3EpPm$l=Lh#5*ogKq9GZfuaCC0kM`f~Iq!NNg-QYu&Uozu)bh-P5dI1e|tL?xsO z3=F;$5K7sOm;ew;Bf$$H6008FVC*fy#1uB$-u0P9kx+K@qN5r3ksB+|jN8*B-zas4 z6tBo|CirOxMHA_fQG7xC7MdOM2{J=Q^gf~kbKNQ4mH>=oEPIWr6QvWEx`6M5U#8I=_+bmf zY}YkgESoE(b8Q&#=8fN!Wd4f}6TT%*m%`1% zvkvkn6{EXos>lr}lZ2_bT@ue+L|Amws@qalBQXEdlsT>ULhg#@knzC%r%U|^D zwOO#7Iumh^pA`zp1gxwi@)8Rw7|KJ5GfRr}&1>UFwF!}<=`~K>>X(NlyJBtpx1rBC zcSKfZeb3y9i_EOhfRE2FW@Ba-*hRL&Awtq!vc^w(+q1#g$SoQZhUyZp`5jhoVRhZE zwEOaOj>o9JKPiyxOJqs7gal2^9?|^@cHCi&<*g4O;rGJs8u=lJRvY!qlCp5P1)|)W z;JPuhAKu^;&A}l%+tye(qy$emZ^7P(^5)1v7*TgwQP%2MM2sufhOD@yMf|Nue?9vt zV=&BotV*@AZ;qm2!_YOmId1dKWL3Xf`6#VVL2cj_+hb#q!^7epP0~3Dt&cl?yfrz* z$r$%Rv9`B%Fz`NNFPMy|A=d@5qIboED5})O11!@4H+95~T1XuX)L zW8iu`(LCn$%v+&h6Zz5lhP3#yAT3bGGD^g^s@9(SD_7sk2c&&%)FFU~!yR>e`ZvI> zm80`-;0!gJhyS`owycxwZN3&2HMt+-#zFh_Izc!(s4!9MAJF%!n-2&cIoh=bCh!T= zz$O$#(3(){2gUlF56#@Hf2;?*zx8Oe^fCqz4sNc=3R6hD_Ap(!?d4I1U2)0|SO+=gN6rqSw}A&2Eb!!F?@s zsi3>|Qcp8U3HO_<v6%iqD_Mt5?E0&+3ehtZ&<+7UHk)f6OtJXjCylH^Cr1Ya?5 zeAz>gBL|!kJ`FUovhLbRq_>>4|0Gh#oJAeDSG-IT9gV_;yM-o1~tvXP~!F)*hY~2_w`24tO zLUvHq$y?7FT=~aIJjTa|D_C7{E1cMY91Zox+pcsuCw37Huoej|>uZ4N!kN|BpMSvHRQJVikMK(qb!*4;E)CXQn&%^Y1> z;-z>;#cNb>D$dz-uQ`86Hc0}8X}&$4NHWJG<@0g42|fK1r-V4sNU1%NP|BDQ;nKfs zRBzh)r5rM}A&IBDQ)EnTfwPU=5#OTk8xo~&Dt0JwOY&|g+>3Er+qTzH)AG8S>K&pj zf$dcF4Wg8?BjvhYJ#c&KQ(ig0pVrlU7UwaSYCG;%N10{Hl~s-asa(~EOUaYKzQYTO zj4}QFw@#b7m1)+ypp_5tI#;q&XB7OnF0)h zrH&kd{Yd0pcA#j4DRhGvW}u&GcZGvKM}IFyoSjhrkHw#u4+t8ynYQcb6arC8h_k{* z0tJ5-{2oD$Z;?gghHV{g^!p<52b!s;<-FECgiaBCr1C%Vc7ed!%lik&M=eKW6H9r5 zY#aV5_$B-@Tn9fFo|@T4uUUTaYlQ`ba0jTVKo~$Nb*g{d`ljjc(_!_Cj*C-B3K5*e zS?iW^+Y!X(i;v}?VSvMp&7N(auYe)lZxKa*9CG`4i++;ZbPM>R>{D|#SsgUj7ItYZ zs2F$Z0w}q?5KV1t`9>wC<_oxw9I>q+u`9@2jnWI{7Z#VCZ8O#z%0E6`kpO#R`;HLNOkuF%R^&974B}f0t<2PH{G6EJ`$PU74@FY@^;p>H>ad zw{wFIZr>SshiqYehY(_aPw)2WWjbiR)&;$^h5~eH9Q0nff6!Ej zUTKdv5oHZ19{y}Vc^EO`=bcXzEI%0LSI%XTgyi1XSnx1>4VCgg*6dM= zAII%}j(THP?6my)_P>NNDZdk}EXw`!UBWe1ukhz7oUf)i(*CM(ejeq!E9^}(E#2*Q zJ9)m5XhKg@7fISnEJP{Gw;H-8C$4Mrghz4j&-?h;MUjfzRwDwyQ9) zC~msOxq}Y%y@DZpH(<%3NmfP=ye^7hxz#;}Q9t9ke%c~}V<#kBXhOF?mG2VVGiHms z3(i&+%`y`^QB3`p-u7J5yS&3@|AT5uJ@$=}q`CaJ=<9EExUzEJmH;LcgOM1ig*1c; zjWqV`^^FvP;oLH2Me_NyA#tXy-zr*j_jG#<3jr4p4873|^FgUDhYvkHc`%5~qRJIo zl?gdAz1;E1vEh;Fi|}Jaf^K;psAHQsoALHLE2yXy zST;`x@9GuvZ`rW;3jgnvb%_~*-~75tToLu!;5(#B+*woESw`vr9DLL#K#Zy zq^9j%kVoWw+TF#rJ?qln&cBOFFI!)Na3XUw)qeK}w6W~e{%(ZY-cVIr1}<$Qi#~R9 zS?(aGP?7M_x|I5ZWvBDED)5B=IFhe|uTwn_Zk#Wd?{vuQL_h$;?%*9K){E_WPtQjd z|9xf7BdwCTy0a-^RYk|v{;)PTeplt)0nTCyKZFDs=;t>`ljhC}OhJ_7pdr-r(x-2| zS3ebF6BZ;$%1}9T>1QbEbY;v4X*9Wdao5x0-EvA%WPiQEI^I}HTLwOWo%8>DcC~E4 zfR*6s*O}~-(@tw`Gt{x(O4|8>VLAPo)zpFRTxJ<{kBIp&iM2xq@pF6vJ0eqvVZdlh zWC~$hJ4Xnxtn?Z%njm(!l}J>F-M6+x{@+HYW2q3o{IW&;-}B5^2rLDA2sIvVmYP6>>;e53i1Xu0W1-PFc@DEpXkhZ%$j3qyTb%9nZTi936xdnK)G8BRL zDOHgd@9wod-4)M5u1qA}74Nd$b^VSb6mjprTEu&0h&S-0yj|+Lz9;W*yW8t`w&*Dc z%-k=HwHD0-N9N1)k;7cT`0>+B1-a8@Ds&7O(mn&zVcG*bv>i<5XVBHlsbnp?PKF!FH&FdU->(HG8fQ;eV5%bKJ^DwSN~ThtB9+CCH66-1$bfpvk?0m^T_81 zy?$T$!2RsfB24%HgPvaN@Xa}QW(GC@9}Oy@HpOia{xT}%feoJ4O)M|K7EW|KD{${h z@=@bm7J8EgMM_=;E)GEUap>I~iF=Owx6=uSz`lb&E~ct4{_EtmM%9kmDemet?a$%V zDiho#l#KYaq!PG-Op?IUS3vFV=C%iAS-o2k*EI$$Y^5y2Q!o?}kZBxj!mE+nf11pywF(-?mpe2QoZrYKrVY0#7{GTCF98EN z)N=c2p&Igt@yQqL;U-^NlRvehA%=jY2w88nn9WP)1*0TV!X`IsEz(DEw{M8~=)w3U zuUQ{WA)w8W+^iB=w@&?J0ou!3%XM+46e&u4=kfX(pJ5DpV=3S-YprJ5d zT**Cdxh@N9ramcnID5036w*CH4((NS1~@o>vk=)v``X4B)2p1!hVZI7eJvH<-uK(# zAJgl;p(J6ZEVtSlxk+0nRekwsU-mT*ON z?eOhV-Mti@YX9@HC{-IeYYj`>%Zk8yc{ftlOEo~-Ak>a#kp2D|Miixf+ImOO_-;*%sl>>7h4x+I+`5Az}v!qRH?h`h-UobzKl7MJ4s;G*ar z`ObF_S*tH~Cc1v$j510=yuN{5Qpr5EYrdAkq{Vcn8aqiK)2!a0t+USV)<#eHYK+xB z11tT(VBT@pT&~3HL91rI9Yd2m4m}=M*P@xnbTr`0Z0VoH`}QG_@Mz|Ip$@v>hqL=W z_Qf;}<`WpYyTIc_ygM1?xKPsVWn!+Q)h?LMp~9_*B+`ZUTl z+n3VOkq%oC?nmVm$rS~Aq|FASs*QDXW`z0w2ev>-zZx(eJ@OnMokyOEY(@QlTs_|` z5_^29$d>E!CXv`9&gQj5-#3rM)|V5jt&&Z&MY4$tG)5s4Nrw?2ePH;?0IeAak&t89 zA4rcG=gb{rNS2g#LUR_pbcJKA2ANGx-Wzr(}|=B8=_EVfTavO zvVC@brjshC3~yI_l03xfx{vS07%q|#T&1M3$}-VD%9|=F8D&W^ zffBw8V1VV40ES8>L~i6D;6RQJpoiIA$q>2l)<(&hMtK(COYyn-C^tsY9}><@4i#J5 zKAa1j9F>BT!$)4_ln?DugOkzE71xD#EYbE|5a$h=8;uT~;0Da;IHy)Ejl6>Jx`(jM0sFS7+Z9c1lbE zje5$sF7=Wlc#V|l7yl`2q~h5jq3k`0e0Cm^01m{Bl=4u8eY9Ki3eJNGg0oo%Z<+LG zi~DV;s6_y$s*hZ!DKm8v3d)EK1%2c)_~4Pt;MMg1arJz&GI)u8Z%PI))#Xjf;8k>d zQ!;q*zHeRzFW&d5GFU}{wewM6AsUgf5=FfTd#r|ipu`YJdrJw)regR(h&+raAVLUb zA5_-YB%}Y%RLM4qC<8rFQaAnv>6c1GioCx1{VL8vKs-0wlM%ez(T2w!0wd^0(DUwq z<00rH7eoPoeU3I<6nG>Skv_WNBjk9)1O_eTDBmt}l;?t%mb#Yp@+0> zreCV7ufKymVmON%QoR)=G-Jat^g;w(nR=sWOk)fS zsUtC+AP&8fP!A7=EOuj*s~WH&WSZ&#Gtpd{_Nm!b4sES`sVfd7X+k`XPz7KUUE};7 z63;_nO;@<$-;~{sqeI-zIFG{=;iSYvBBcF-L6T@3n z^^k&bv)*b+kA!Bu)dnG=0dg1~Bk!&rJ4MQ6&e_)SBFa#xy-@W?MKgb|mX1uA@;RT1 zWT39F=7XOS)*nqsHsiNWQj@5BLR$F(Brc{hyyl#^ zm&l1i%>baZxl#=quXekGkR=y01ioPm63Cc6wn!>U8r)H1B;!nR({hA(xWjYw@h8Y;hxqAXaq*1 zAeJk-vz*wzqGc6O>^n#u;VBCd9Px)oP`;y{)c$4e2G95HM;?;4n*Kkoo^SSizeK+` zb-rJ!%bPskucG6dI^Qqe_syU07w`MD^LMi>+;Y8Fs(h+c z`9l~4$d6BTDidM=smdgzOavi|&zrpW{`U;HIOOww7q`e?;9pN5HX9$AUvWc4qE4zCw#t662;)*if0h zTEgsg&Vii|M<~re{E91B;VosDd{1{i>3pOdHZtN656B2TO#CEf5c8+&nCyFyA!@LKo`9vVK9Wg6m%riduv1yfDt2fD8f)C_goW#qRn+ZnRBIs zZ#82XE%g*}0=#%5N;Zf-bQlSdTL)*JX9~IGp?J^YcdyZ%xkS?$eUq)w!y~?C_@7*u zAEN|CMJNyBK{S;Dyd@OiD$OrwH4^)H(P>vi%s1*CMS(`W)2iRrWg-`s zAS!}~_^gf~!;V;^F(ar1f+BuMmANB6fJ!G6i(g7oVeI;y*bW{qeq|6B>iW|4qu}7G z0pR-6`)_BDx4o~^j=o#e)AGaaR&{dn@pc=#T7ID2p5D%Kgx#LrPCK+VmAS1c%xzN- zCIUSN&_Tbb_))D}76P!oqrAg7Kqxw!6ue%Lj=rg^N6lkXzBP|$l5Ynmv3zShOZnD# zbnK@dKD2yWqNfih-yXrHPhB=uA2vkggA+0!L~d_7Q;LK%^4XYC)-2RBcrOMMzr6KJ`2@00E(_ zEG;E}V87L9HTN3_`_1-#zH7B{tyOEtER@+?&)OYEmvwL<9rg$eU1bzhZZgPCd)PG( zxcoyf^e_rjM>^%C)9j>3*O^TWAI4(%=vj*4^=A#*&K_@jUmscwuO%1IR(0}W#qfG_ z=4?-IrwTUZxzapH?al|Y?arNecRqZUyYu=p>TG9^x4o|qy*sZZPtI0#@?m%9_2!e= zp59K~os|<)vz^+V+q3P?2l4LQdDz`~J2vO_Cz{!w9zQsTt~3iyX3;C>tY#~<=v%Wb z`gXkN51!kiSFxKlKK7NZBUP(QIr0jV?+6b<5sb%395h3G?+%D^663Z|V0e&a^Z?5H zsM$@!e&-z%qeo@d6n@ z>?2pMgK?wYmPO{{Ri|76nkj#|=4}3Q?ZgJuDqM7p{GvM-c+Re~WSzBEfQ-`uLQTtw zIi)AsWp@}?r%NqBbc?ABKTKxpQiL_Dt>MirL!l47t8_d?+2fbU2@$(e3RJ1k1s-A) zaFS6#1LWZTolL_Zo)zxrVrx`?Un3FI8Br8Rg%~SI#$1DxLpkqILPV`q7N6`#o@hmW zT_|d07(~JHh7`agc2frPQX-#+fT2;~$s9(I0VhNvt-=hwGuckL@f;iO)qFk2ZTBPJ zkufAa#Aik1ExV2T=;cJKi`6 zU0H(A*kDF=@FNZ>tD0>Ke_i3`A9slK`2dC2RKiWET1uZT-n@HrcKN3J;q3jJFLP8- z0=1g{%2B;jZ^ZI9HQwX(fBlpA#uxeh_t?iH=%q?y8MM{!$Ym_~LGYh+3Bwdzwdx$s znOv+i`$xtSftrzw6exO~RzX4+z8rjVqXU7c+*<$pB@kap?f*(cby)J1#J650>L{*g zSxp58#Gk!N*Uzf;N$3c_wq$jMs3uuRi$V1{5fQ}{|8jXCLIj`>P#6-Gd4V`i#N~mD z9kF{$j!d@$9O(1cdkBn(i~AV4=}jK{W9VV`RP0D~U2W>dH|rvX=2AUy z{~j`DC=+~~j$rVKGQL57k^kIRirw-nr%8w^LaWONi0YC}ObZp}4@9Af2Suknd+IkBRWeN4`TMpP>-BRRGm24KZaTyt~py zH{oyJLvhu67cmr$u#Z%l7rl6S>o+94=52|PG}&B8Jg*0x>npuQsOM7;8P{VP1dv)-U2umXe6Y=KW^N`BcLM8NZ5rf!M22A8LdYa#h=#bpR z5rH#rKyrTnDN+aCDSKq3$!RosB@-X%t>$|QKIvRI4o39L?sw*&dtzrY#FrY4+9)od z?+j50-p4p3J1HMi2_nQw6uqH(gy=RPd?EGj3R?S~#^0u?u*lP$ov+{=hH%6eANUGh zBkF`$CMDv3$s7t_cfJA<(*@MOwe(*yFpBeE;#>Qb@6TiGa)~a0hqT=eN#wclvN!TP z#eI@b-|$83!;#2=;PH*0k|=bLj&qVjHTL?iSCvo>yq3VZ%2nL6jO*`L@QYc6#@Nr| zhrw5HRdFC=9x|G&(L2h}sG`Huj0`9o`dknX(YoAm`Bz@ay;oM9eiV;`Nxy>Mhe#AlGENSk@^4Hlk(jVD=S~$vyw5^} zWNk!sA>Z(xAgw}sDkstO{u{tI3OzwI@ax4p5D@H3=_onNm@#8nsV|Cpls=zjRpftGUMuatA)qj&I za&ipvvTWU}R5#{}@jH3`iz6V#nJ#cKHGt-oC2+cpoNvae=6J|ru(YSRc z8x%5G3c+7*#XXR_sS~|WJw*s4H9{4Gr}Eip zjfa45VT7v2^R(av8P_MYV=ktS$e?vj+|;AgfjQ|Ek1h-TwT05d;R`RedNll2qQj0*p~@)#3{=MHzI$W=jo9t%%+iv8lRZ8 zhz_M%NhLg7p=1rO`KezeVtcQmsg3Pa2^ko7Lhq__9CC0mv?xpLroG0n4=(=kCdt)i z&bRE$rB2YcWG9kDZ?k_}(0Ix324bFO|K?kNz|RdI2H3w{{L@&rQuBE;bA0^`bKJ$h zikRch@v+wW$J#3W{&kzGGJuQEZ=2RKl?!*FBaUNqQo_b;>cMJ~!7;U3_V^&K+q8TP zQ4gKDqle{J_0egaS}Whq*oU?7?~LDGEhkOhLuZK5Si0+00cL64qxy`11jI)PNK7er4Q;LO0tf<# zg+AJ}zJP!O330Ub!Ov%DZEO|X46!Fq^0G9lk;2r-$g7fa%ATt~00i;z`3|^|+EbFb zsS1>w4;eMGT&9w}pmf?wl~W{@+)qxOIFWs5d0! zI$2}#^bEzMWXLN^(u#c32~E>xv{bK?Gdlm0nB~+AX8NzY5)ij8RY{!5MINXkh&kl# z68`Zltu(aEf03{4cq{nC@4LjI337Hk61nw(m@R>y6HRFiahZ&yqHaiTYK-hF`qcFD z`wqHpS9hi>I~@vwUSOL z3gnjavs!^t8LKsf$~N}1B*_^_dM5aN6fyZn!Xc(XFE%9z8)ZCpqLk25u`G_lQZ*rO za<2RM?owng#hgtk#cDs``*IZM^m7gh8SZ08n(zp4i4sa8Tbd6>gfk(Rj8(BSdHov2 z10T8OoBR{VC(&RiH6|3XD3CfsObw2Y$0EvI3trUJ!-8R*&l`Ccdmz-S3N~LLwGtx)w)gp+)AMyr%V? zon{@J@g<=Z;6QF0AsJ(t(Ho|z;Hy<3WPfr@mji)16D%JxWMVqRa*f-QW=?@*WJ2^X zH9g88gi_zsW({GzR#qs{OLU7Ft?#rX?Gi2r#N;cS+VIuFfx@jqVz{`bdQ~kr!EhcC zIrI2nlD^b;+I8?-dU3y>xVY1^nO65zV_wD~=tJ!B=^GJO=00LXDQuMR5i?E|v&vM4U@9}8$;lWAIMDR0c4z4#DnvZTM;V=iGl>4 zr+5YG{ovJEvL4l!Ij>55s9zK;Mg^5+XWV4%mw%a)pHNOjHJgD*d=A9)Q8H^&a;r6D z9o^Y^YZ@-h&5keUPk$gLOh@~CvFg)Vp%$y~$vqC=p}Ac#1Ss($o%m2gd$nn@c-e?0 z!w_Z21R5%De2g7ayDH~AlfP4&!x+mACq7{sf?No_nS?oSA$R7L%56?~S-XnU_#GZ$ zriuPTl5#)ULSoG&Kd(rq5$tmS8W3p=4UrT-e0RpsGwlDdP|&M^5g@-H5S^h4Ru9xc z(gjX*Os{#{Bw{xR4~7Y+_mGeJSQNY%!m-kN3NGQt5>pxou_d*jNkVdoZeF7?@`xxU zVPF!8bU3H!*vHp#?4wFAF1MKc?X0(nYAQtF4~}yoa7b_`cYseb`cO?Xzf5{}Ps%~T z*2-{K1@z8$kPCJtGUps7XMayDG@%_8p!U-cI@e!z`TzGopLi|`(}$ASsSQB?F3!rz z0jWM$_HjrV_{2dq8={fMQzXO#>O%ce6#j|$e63#G!!Zm5b(?*v&J_RZ` zgjN+06T50+zqtY=%xA(Co}9kO@fmm=@aftQfikf7VW`DN!3@{M+9*DB^6kzrAMIyM zN9LN4CdTxd^0Q)L;tIIRyv`T*saR^6&n~mf47Pl2er=!Q-P$0+E~5JbDb#9|!H^5j zk^7wrjjv_GcjT%~D74A088Ky|TUrC9m0auZoI1EZnLobE{hUbgG3? zDUbLHkB>D~Rl6xf(lJiESjW&>`$sW2P#jUHy^M=^UGjT)#F_6-x%X&k5gSGoN}b3O z9AhYl;#h5~yir}qU~Fm+-C6)cb|<`>g#(>n#xoxvNbkT66gz@T__I{;^{*1iQZ_3q5C~eEzNaqQh&`ws(`(DJ6@PUX)qr$JMh&v?g67T#A zg?x-RK3_{aJC~v&%T=K-*%hA&d8Er8%Ei7}{=GQIGgPrILTjO@uRpGB2TI%T)^~m; zH)yOCuf%huD^(DpKE6$ILH7{2MnRGi{|5RjE{Z1Kkens$`jX@L`B0w|#clw4h~Jc8 zPIqa~IEN6^ze;+5r7aT$K6~07?UY+7 z0xw<^bm+wk)4QT*J@F=AyeOo|ix+x9<)>E8qtt#0iev>cb^yV6|NoA^&(!C6PyZX#0`UEPg$Br}oZeT0%tIGQd`cE^tDIJ32P)sbeXl<% zzb;qm>5CWOBLV0ZIg#@ChCMQlD@0dyjmA)#KUBnpwj~WbqnNHJLy2kDT1pc7J z2eshk_n+=7B&eK%iX=p(3Mv6)Lw;qScY0rUy7KMI_hpOY0XUqIH8A;^sIDsu)e9+8 zm5N?Ta*me~9ZnEjDVGdk(Sp|q(Ab!+W?yd+q(7mp5udsE;5|2ioY?296lT=)?m3AJh5FjRqe*a2`vB3T%hG7`c;Y)M0|DjCX#Lf;CpuWiU9 z;v%Xl=BPtPHK{7(#x1Fj=t7WK#YL3tpAu<_K*oYGpY8zB-lI<(tNa+VSLZ@I5 zw8S0R#rM32r{GiN0zvop5Hs|P@1V*T@ks2YFWZW}Xw{!_>!+a8_z5Sm7J7@*L5ndt zi|0s;mA4SnL!1y&J-RYUTHo2p@2g6s?+lT1omxs~|KgJ-b}A1IpmUd6gWpqd{+pC7 zX}1o!kmF-%;+r6jt7rexr&JEiiH~A?n{c9mLUjkA;VXOgFAlYo7xQmbGss8UR{9W& zqR)ImrsW~=Q56h1XG)ACS0)FD=fBmpCOLh200;}@#X-KD1J$T9JCp0(!9l2iqgo}T*8H0y@wdggnIo+e00js(X4iZD62JLTqY)f zRG*t^Tr34rj>{>Rg!))Qi`GR%0=QIq#0_5{n7UUSH(C&JE!H7mA@D4;5xWW=y&k=n9qT&$)%x~g&5Bc$L4 zd7fUhqAM(tm5E~gPuidh!f@4jKeLLhp{hH91BGC=e3Ss zc$VKX_3QgFQ!4XqD3J=(U_i?PzADry zN~+{EgBbO3Q~rBFk^i0urq|P@q;gGS8|-rc|fcD`IukHnHSG|$I&rzff-t7<^~agnBs}{Pkbx3>&2&Qq1IRS z#9Amla-bP_D7^%f8-=uxy2yc{+5~9qrzd@IbdIR(w1@D2bIb!(^KwNQWT6di$*uI&KD8FF5J#2!*Q5CK5vT2kUvlUi32YRk?S zCpgIT1sH#tw+hLcEZpR|X*1#WYI8i@P@fPR->Ip_n3)SQN6736GBc%%O_jbe41H+i@pN}qhS-2fPGUfNwVw>vAPa-5Y{-j1?g~>V%@C zmpFn#*A4lJTW2tlE}p_hAnliofU9QfNMxjMo;Dk;gOe*MObVg$ix)Dt$cq<=i+x?F z7nKN;Zre#GlrNAtiK>e~VBbBH{kl^AlCdw^RYUZop?i|nF#?Bp{7fnwi0V5#uLams zAs;DEQhhKV^A`hA?ONifI{)WN&SFpCggAOXg%QzdP7H8B->t_LS zIY=BC(=u_p(%X?Sic|2lrnaTzQQn+rt*)NDJ{B9*xCy;r2%E;X9j}{oVhjGjcc5Oc ztKagob_y~pMm$;cTxvE`Eeom?j78p;9gJsY(uRC$Qc&-vxl3yIGZo~V+JD)ipL{U> zJ`93d45?S~jYgx7^|hRZFHvyuz!9bovIVw%oK5e1Zm2#&y~LQVUM5_c4~y{9Ev za_pBURl0hYMX6w7RNASaM|A4NmAN7ARa3n_TYKUYDCM+{b5mNjeKTHZ1by7sxi1D# zfCXJd6TD?Q7$VMBqt$*gmLea5NUK07bBxg0d})`eN=zssAB09y9yfiWeAQLDq(ZGj z+M&^K!LBBTEfR%7A7OaOsZCH&SK6~wmWgqze!Gu+BQInPD!$Kx3nc-jT;J#3Z7{oCBZ}5=sq& zV}9!f>@GlZ>K;@|$z4wUnO~Ud+^4*C;}n@%b&y=)42V=--FShjb*ou#rp+(Tk)TxJ z2C2@@-vX7jq*y%hjtLGDVYij!r>6#K>?%tRK2zz5VxlPs7VjE{XN60T@aDI7isI~JmV zZH;n(viZ|VntEo6hr(5=bAF5N2z}M%xyqzUvJfuTme>RN*Vxjaxa*cf9;Dw5k{Lx67&=}~eC@Rs0LF&XxwEjfo-J zO^sNF@p4rBYe4+O31h-qW<;vWtA;IEj99zq<(5}$1|%{ZpofOg>nGfA-`rY-Dkq>i%9P32E#DS2`JSayXfGHZ?o32T(HY z@%+TPAaP6*>nY`|T|7z&h($*H)B~3c`mP8R%t(s9EN&r`Uw@B+=3)J?-jFp18ug~U zc2aND+tSYdfiUbKVHQ?i6{kDe2)$DSSIU4~HzE@$f}t~va~EsXa&H%$^*fI*OZHc%pxTHB{;2raU4ivpEPD<`0k5! zCRU5cF-@f#L6R9n2d^78&@l3l$o7Hig#`_+5*~%%I_ZkG(`uzoHFntNOBRYONP@8m zU?Cn1q&H_GO?{PXgW(aXgHP?&LGy4T?$!S)#}Vhlwn}UI zbbbmB7k%s@x}Qqiw=e!*|AP{Lfp1LI|27*(heyq{{Q7jeB7dsqp<;JSMHP_u9`FRC*Ei*jvAH3YyjAY9cNw zr=U_xln=F9#du8OmCC8iaGP8oVz=Nbc8LFqcq9}q_JbbWZB#+4*{p(Ar&9&ZJ+o<9 zB>+^;lHq{=l4@^iKF8$>e=Zh-6?N}hH9y$&u>NNJIm4jPZ6JD1qtULC$h(5^x{4XQ zN%F_81RPxtQq~QTN1)rKkuZ+-Bd;ssTNxY$U5XsyyYyZkT>tv=b0zY{uK&4GH5>FG zr7!>Ux$?K{#m|*$EXl+J@n4@S-#)b)^vj;e+_0N{F-31RWfFBxnlmw1Cx`OC^qhHk zg!wXSHW~#pDJ^&4LQo?$hjd78#1b@LqYQ~yjCLbE-*E5`bjO!=B|DzXV}dWfnAb4# z(2wG{pX(2Dix!n=MXx9E@0T65vMR{MEVEE510S)sZsipCk>|-PAq<8e3Go;X^Z+YZ zr_|uI>a-8qos+}P!Ew7K?t0k2PWoO6ppQHnGp}CF6$d&)^bwB`iJ0+4d5_Q7574mS&IdHz=k2B zL*luz(#fPJ-ox-32@>*kk=>0dgAlvtgr<8rtek?j(HQT-q4D=^rk8j8?_ATE&vdJi z{OvOoj-i)b9_baP$P@(FAAFPxzgcuWy(p5-8!Vor4??%hn0ugZ@uuU8->Q=v3uQ3a zq{doFW4To{g14_BgXaYMLUbClU*XgcH{KJm>+!9`7doj*>Y0!H%NgPYgt9*76H=*U zFaAh;c8UKKO?Ddp%eXIMyEE^IYo)J<;lGE$1hVrp6Mv8QFHImOJ6`#)f0nsH;uj{+ zjs`Hi_OPGfr9Sq&v@mEltDxCDu7c+A3FqhLaf;>p{5v^8${RMDk*&1&&T!J+%L;}v zJ`$VnE8;~XKTTIzqT~c!y4ab8n8KY1fp37ymyrEK* zcGYUZ#9y88Ha}J-$E!lq{PJ@JIt-5!!M=b%rwr!BCD zU>|H#PZ!4-4}&F6I#qCT!vCvxCgX}rX)Xa5kRh<^6Tf!wwR3w5LPP`NQ)HMgEZlg0 zaKl9o9zh}P8_!GI(9S}S;~CWr7J2{+JuLK?4n5#_FwG>OjQJ0$RQUF(DG#E4 z=<#`kXJS__a8i`l!%!@k^jA2`rvw6*U+KY=O0=VlL`x~ei`>`RIVK+>JQ#|(YBd_c zZ6zb>4wJv>2XvDil7|LJfFh$~(8B}&jD9S3=;c$HyQq@#6$t$ycF~(r!0t-!{R@R8 z-`)t`7Cb+~e!jJciPw-&8yf&zhhS{qIL@LP!bAC>fvN=y++i@Hz#pvzF#ss zZ`t0=cX*nc4eX;E)w9+lEHaC9Rt6_G_+Ga9} ztB#24>i0yC-n#?4>f+-5Z^d5hGai3aq7KX;VC9yRc$DAY12qEh?OVDf8FMD2jV$tB znMYJN8?(x4V*zp^{U||WYEAR&mqXmoE%sQrzg2@M_2Y~E+0DZjy+FDgYQY{thcHBL z!4@Vrs#bQp>1w3lM59P%=FLTR%!D+A2Y%A`TX>5z;;R>ll*<`%@FA%!iBrtnLDiup z^K|hR9q*l=zT~=p2X7}Gw3786qJIaOl`b-#3HHXCaJn>lxkJld3?@@R`dWkC>~ z!jw7qxLHui$7lg1m<%Yl4jgWin?f-ZoaAO^XFIyxyH%x zyCXCr;axYPaDYxZ`*&T=|BnFD&gE6IdYIc>4CX!|C|l$l3bq!|(jlL>XAEh#4qC2R zjd_?smd6=jnbl%8^M<*}YRq>{(iFq>82tPA8IJWeH>SgCTbM4;&jtbrM)P=cYyHHQ;)zpI>_x(|BEOTPNQ(WdfmoifcKf`bvL9hwOZYIAaDvgS*`9&*n1(Z?lkP3 z^>Z)4-r`$F53SU-2s^oG5%w|&8;%FFhPH=@F9e@yWnAx$p$GOy-ve6IQ6=0Yk|98B)I3@sS2NpS+pgSuDezb=C4a&A4=)iT@tnZaHv$Pkzqs^X9^ zhXbD@+)MZK;n1EV;=eM&Vy4VgNk~z%Va61d;-KRU4r(1_fKbj3e3-{U*cFkyTFT|v zMuMWwBzYac0a9H*GQ{cvp$ZO{C|TUVC-i3oT_K{=cjR7Ski{Whwn#vtPs$CRvO8}Q z!wKYWIOe3+vAKcgR$)Obvn0T@cxL0Ni80N|bLDZ(z+Cb`Ekq-^{6GKs&wtk5zpuT1 z4SxRlbTm564zWnts-~PV6pH!7MYIV*L{T_SzLi&`Q0QYOS55wAiAn;P8#!em_6OY=cB@<>4En7TKjq>-Jt z@`?Ll(!{NMdd@oy@D4KMJ9p1{jv842pF1L!Ox~;2PM*tLRr+_XNj$NWn484>T)p0o zxjM0PHGQt$Y{Oi={qUTpXqw#m@u2|64|7R0J7on^_A#vznUSN0*maS=8VLW(uOIEq z=o=-;ld`j>811Lk?uBV~+R2P_c6Ij)D$^U!h)y!+#dr!yOZ z@x&F7PH%Cf``wgX9V^TxKQ&0I69}I68pyx`m~Qi5nG&1Wy@)ir^HNFRr$&e zm~0U%UU544-K(5Ou$cf)d(2Fr(UzFmS~_WAk-Jf|(K^Vu939VZU0b82thnaP%%-yZ zL2%A9&E!L6Tu7UtInIV3KfWtJ8&({YO;-BkPKTx1utJ`jq-kknM{#)4Y$eW!`A*Mr zVti)mml^aa`}G(5t38!x^(l*=%P0E(Q|fimZGc16RqkD~cKmhJL!pltq8tkdIi3du z^GbIq*oTC&Jx~Lp=HAMy$`nX){=o?;NCwxa>mf#w?}T>&>pCc8k|wgAda?8`aWk6; z2{NMp6$%;dV+S$>-th0;l?ix`2Lvg)a4-nb0A@?$6^Qj6^=;l|z7+ZWj(n#=ogo^* z-%vheUkU?;n1{X%A@B-)|{PQH4`Do&+X7T@jM<^6t;6YK}4Z=55 zR|n`evoo54QgQ-^ieich!mgbsxoVd@>A(Niz|DgUF#aVFZaey zEkDtdlxwzKdt1Nu7F%^2T5q$echc)JAg*1NlRCKFYc=mFJ9VMv9Y|NSO5@~;OE1xB zn#^yBzJJNJ+uDw`EB^hmqZW<~c!&{sT;zq^)SB9TTi0>de|S-0u+klx$mI* zAC&l$zR_qj8ixl5;@?K2k@@$qanw2bOY`8UbJ%Pi9CTWLX|y^A2d%$=#)`%`@h74T zhJR@+{jKca9?6g5>~0tPLBx#Xt!AsP*otw83hqJQCk$$ii;V`EQ-4Mi)r+vl-ugLR zOU7v6FmwxabmCtn!-bwc#DR{V(fe;<`pL?`M@;CKP7a#~`dSEs;m0^+ zPmxgCA`%^H1v8`+R;QrZIc_x%4m*d3?Y6RqEz3T+{?n;%R(v3qtneW%S_q_b2tS->X|TgbM(2W^h}LA1GL^}_oR0*s}%O6 zJeWNf&fd!2DVn;{koGA-?GwV<%L3achqjkk71F!;WPhR0J^+n+r^^4O(Mb8%?1}3D z2>%56D*y}o66pp&Y@>hqA7C09^aG-}t^8w;|Fiem&zXZt{W%J32}XJOc(@GOav2h5hTqQ zs~xu$d=HU?-ZVfy1H=cR$UQAMdx&B%4f-J&i7OdAzB!bgxwItAKCm!bUAHj%5yNcd z4*WoHr!-0D6fiW|;pdUU^oJ$=lf&uTOTNbj)PL0yIu_m*qg#w#*k_B;w-KXnT3da* zIE-#jhtZQ#wLTEcE-!X!G5E8=;0ag>ARDeot>}=*Uks#Mzx|j- z*O(uR8uNVC&LJCGOtW50lP@U0aeQEL&Ev&28QtTLBm@XLL%Ai9o=_;dRe%`2}*YFl(a&j(MvpR~VPcx6>a7FuPR ze^k)wJeFI4^ufWcbfxUv-o}hh0UQy30HEh3*XrA(g^F|K&yTE{sd|SNTCEmZ0Z750 zC?pnQtrTLVv{7Kucc0&`>$Vr_l(x77bZdmBm1r=C>OIZCbAsCx=}GugnmQ zYw?Q3E9&+Z_%}VMTT?Z8BGxb&3Z`nQoeuQw1g@&8_a4p<3f$l8zCJogl+g}Z->7(8 z);G$cqHOaP70F8_YMdot(aDKLMHUq$q}*my6k{S`OPRwtNmp0_bHJb&=j?s<_kf{2 z+6SqA%-anC#}@BM+>boUDyyIKu;HFFD5>TS`{aKJqR&%NXI2L^MQCk5RS}S-ds@qqQ^)O5_S+! zlVzcmwSMOM{A@ARj`0bzR92G0K36TP)zUX|^|G=Rv&@=VM&9~--K?oOw2LcFOs}6c zt%6o&g%nhhm|jO~S|zPaEiC|yJLysxlPhYKm1rm_(lD{E)}+#fB~?Gtvydj+ED%|^ z^!2fiwMXB4buo&IR+LTIgz_KegkCDykS_#MUJ7JlF_6-7Ad43SDINHni-BI_;MdT* zyz`x6{GF@mF|DdcdC8ZO${rJHdz4o9C`&A#s`8Po^I=xKNR|3Xzn>}hF{R)~y5vWu z=trjPN4oGwx+q7cEJvpNN4ic&x&(+(1SDH+xu6ipw6CT=bC~{2`m1J9kaSs)bYT#) zG)U2e6%`06DiNYnSp2Lr=bk6kH}p%YT*$#sKlZMFq5p~{P^xIiPOkn#Y5fm%W8d*2 z%225PuBpFkK5-G9TKYr$Os@ZNaCp>dWa@t$95gz%{>MZ7a8@s@<1wLjXJG}88S8hH zRqrU)Z_ib^W7fJINSTuia$o!WWf+Z{#{LNp+}ZJN~mkYgnu^EY}zo z>kEsuh5FEx(?b=j3X2tm$w-TogvBbtDGI^~O1?_qMFZ?7$EeCty=kOsEmHxOC;-dV ze-o8|rK-PT#or{g-|Q8*o1g~Zc(99wmhHYPiph4nFaPto!ewgL38GkoCT8&K4fb7f zL&Y5dZp7u!mFkGN=;h}Mc0KgD^6k^#x?SF8jMlV0QSTm^GDoFau>y;tu>oa4ch0I< zQ0%%`-&LmVD%EwBYPu50^y>Uq@GHX}{xe;#yHL4RrrIhhMmbHbmGDooN~=JjRjSS^ zQ)U&bvWgX11!}CMi+L4RT7RDuxlf;^rpGvz>WRv}t3}?alcyt{ z646sQ(&nNgZLWBv%~?nK!=fWycv73Ip4H~e)B0*l04(n|ILp|WEqa{;y>1=wvjEis zRDFN*s;=hY3ZUMBg?bk1&8F*$p}`1^NO;$Mk49%>h&|ZzP*=DebF`rG0ioSsct^1V zJ^eyIMC!o|GLSArePeVbLA!OFOl)ILJh5#~Fu}yOZRf<87!yp42`08}o!GW*o-gmc z-@ERQu70ZfPp|4)wQBEY*M4l42PJDpeESqSw|lvA0%OlQ@H^F#0;vMS)RMHYjg^P3 zk?p0C0^(y~0k|RKaANKzxnJ9L3NgtcaUYooP2k_}g>&cufO5sI4`lJhfx8#(1cJPe z$I0Ig)PJsqMZSLdh**x5FYI1jo@l(fpVc}yFi6KwC()77@C`qB?ya#SCb+WcwM(hYRaj4cLr1?IBVboSRMN}OF=ntO7bHsXlJOvf}HtR_~i)wDS(v|1lDSnO7`+!C& z&mwJk*VD>l?j=Rl$4(iv;5hg=^d67>+1JGp(?bjzo2T>5}`KVO-)qz~AF-R!xK7#N093F$8tluf{+LMYE0a zNk>CmlZh

>bY!ZHjYLL^3wIGGaU3ls@m6Pb6)(>6@%-VJo>&f=K1-igLS&a7^R zN7k_`m0mOLcYKt(a1&gLRT_goR#}u;xd*Ts^i;6xW^`Td%*|ZtshlIpbL=SK2WHvB zd0w34lF7;Tbc5pnXOA@|ZWO0%5?bpK^D&z15g%>C%*9^S0rq=KCsO71rpoKzksXnf zMozYVfYfhQYsN?K$k2pQ6!qt8P?ly^*2*W&OLs>}=52+n4)fYG<(AVyTN=q3E@JxS zJLQ?P&dv&;;q(#xY?*T&M!ds4`OH#*%xg9FzLP!V%zuLLEKdhJx(~m4zRu- zxL-DH6s$(mAMy1ue1vww_&p;@^GZKHqLH)_FSXO#Z!r89mJNMns{v z46*;_4?|$1@fOq4Ce3&1(guui@9Qq)eWR>rOw6-TWn~Nd2@}s5In&IK4Y__RI<#yj zE5`mvFYdfM+TK|JFh(Kd;E}Jz@_fzsGAULul=#Jgs^v1fmUP6tMF)ke`X40{F@s|S zxN+n(U;t%O&};qSZ<)`G(x7>foW~MW`XPcTgMN+IK7;_(Th)s39}NO@8EnEfa=CTm*cB9eUia*(xe7Y{W2L4+3z>6vfyY`7=1ubx ziqsve$o@KGHB;U}L-w zGWMnj*>cN`tf?c(GWyQFtXOPOxyyPye-a{R1Ws79*upjb-i~VsLdh_-`^%A|(NT@^ z1kx@pcRMLH;m?qsI8gQ>_nO;#Pkgg-F;nm%6jr(Ix)%&t@@Kx$Lz_stn|Mmk1Xp;? z@r)k;ZMgWPxxlO9ZwRPW%@KS6+_tp%HseOBF|*l>5fZwLeg-PyS@=i!u(%+U@<0A! ztQHk;8@oDtLE00;dqzdqw1V%!UkD-m*x|@xtS>6Bz!S~T#iTFlGkN*q!c*{I1@la9 zD>DdMB?yT4NcjnWrTEbh%;&zd=H+tklW(cJz0l;tW+v%uzgrnO^Fs3{JQaS}D1FUD zk}A??Q&cpjIT_9R{sgnJUa%Svg4{yBoWdj0^``r*^EPxS`y2dmBYcvN(H3t#E{PpJ ztk*t0Gk;Abd9O)#eF-sQ#v#c1D=k?x1^QhvegL^VyYb_MHj7jFEHwgh20qoJ z^H#z#thX8|xVvvy4{Ks01i+JI;7QIwgn{?CC|&*d@0mud0^*tJ0p5mup~dz+a_R)m zxY6AGjl@{JH*R*Sd0@M&cY6qyr`?TZ7vs?r6V*Y7*xu|ymy1v9KXI#-xzp1d zM;*`F&ei?Y$@Pw7nDlq=$-m?4wRxF3Vy_bV%#QrS{G@wHQ;ON0=?C^J{bqI3HsGMN zx8)?UGtsXCIIT06=1}_OL2qy7hm%+QmTFOPOgD7{!@jqweVx0KOng{EEmo6-#A>Nj zmKa@cG!rbF{{p}bh-_sBp(^v-4QW4sv21a%<%;3YweATKX}_etYhFqV*;b?d3^U*) z2%_YaNgPdj4EMAbc6kOi(_Rg)e>x^GDaL>MAj=9?ulz8mbFq9ECf%_J0uP;fHdfF? z+`}xZD`lL?=#73Ae;iG((v{a3aA6jfGgF9E9DbW%+J{ZN`ZF(H72Z`Y9PxbHjr`Eg zQ6{H!25oeUejZt>nfi6@5}p08yQaR%_1o|xHS%G>Mv_V!lW4mE`GXwz2IOI`aXWUH zBezGy+zn4f1DN(D)tk;M;rudaIzY8FKa1T{N^ zDqk_wtqSeoUR3s#dp>R>C5zn{N3#KoPeAzfA^O3ffdy9%f@S$*^`br_%k~X;T4HqG zg?6oC3FdS$)2yrFe6`1soo18uG;@Pba}&o#Z*^sYShpWNwP|_4V?zLkJWum}Ro`qR zjjanaytQCT(V?=D7bX=bT`I)6&_+68C_Wc*->mWeU^F9ayA)XUD;9~l4e`ifwK5Rw_{q7#+PJ;O?2Q!JA&Ht$>`f+LGlBjp2poy*|)&bAwD{tNvYu1IsGF%r5 zMZYqb9d#=~uEQ#NC@*$?W>tVf#LUIcfeyGt4&uc%DQA{4w{c*7%_sDcx&` z@ZM0-VW4`e_}iBoy{=-S*H95cCm+t1lO_uS_54eBHiWvQv=5YKyyFAR=c@;pt54Ie z|J*-2a8~C0keb1|vC(KI@irt1d}zdiL$?9ZifI{vMU(+rwfC@$4x@()q-~3RX6!^8 z)OE`>W_46cXDuXnsZ?t&D6M)p?>q?JoD!gW9U$ovr9PxX-Im{qkiEy%xl(2$8uj@W zjXQi3|5Gpq=weIJ9KH#101N6@v8VN_?Vg+|g+Vvc#|Ra*qI6-AaWjrrw!zVZtp7n2 zm^6eE@cPdz#1awC7`FSYn_$YwW8-W8oC>LIt$dL`P- zbPK_loq9-F!q$He(Q=+-x0~)Xs7YK>?AKn}1$LlgQMe3o=EY;(GIw15qxZ zf%BI3v}{B)8ma`jhnB=nmK~pP(5DCcnSY29;dgPM*abY^aMTmPrTpEZh)rseh9$ac zFk3T?Lo6#S5~O1g7xYoBmJYSc%@w3uD(JhL369c0*A>Xo zxL1i`=As~t3W~M~l2`mT=eh7?uu~WBNs4G=wC9o!!hjW^`e<@mvNqoVuEKMDJR14` z+6C?|!{IqM=J;V>;g+;y``kIy!n@6r&V{1(nap(nwmA`x-^mJx0yoGCA6v*Je_}kn zazQV&bt!f}oD(&74GEb|d3=I97qKq1>5_0Uk(VuxWw`*qD|*RE5vy@8^9%xOW7~E1 z$Ws)6cpGdc4te+!>VpzqDBz1#Dh6DK7Vnvg#hsZ0+TBHWrFkYDLInR9>)0NrBajT; z!#=xBDQhTRj^ipU4x)W$NCN%qM+WIdom1zUySlBy-Geu^Q{}AJH7jf06$3nhy6=K~ z^ZZdQ=XAsi@+a#S@#YaiKIf*~9YznEB*%_56btU-hu6OaX}PrFpOz;jL7K@_#DAaY5JRz#ellU}uAEDPIF02-*gr&YJS`n?OWilD z0$zPSM%R!%zqKL1P{KG65Gp&4Y>^0$$uzE==LvnmI0BrZi9~#m07m26;6eN`78F~J zSr%y2|CF*t3ylDi*|yxDSAc~g!=ydRq{7^cKyXC6akk*H{pSTSnXQ?;1f+Gl!lHn-++q| z&RUD}^ojYbntQ6%MoRxuIe&e{bJI_M9e|0v@SQ*@$Ir6W8eDl2PP@~Z43N8}u5FEQ z+wszm!_pHUvH{-RWsPV0ov**!tt8c$Z_ZbK;>GEmt={;Y?(k;RyIWzd$Tzqojnka| zxOz3cnLUuPNSa}i;>w70XYI8kcRO(n_^e8+ZmDW(r;Jp=P`^T#sAc&{RE4U?1v$o{ zX=@!0d}-;VAyTs?yr-HenRKY~cK$Xut6=g?QCc#qIGOvs^iBRAOq_@@1Du;nuO4Iu zT_Eu-gKtY_1&;P^x|-t3!H42+SB!a3emN`{R>_$?Cf7rc7&Ta`@lgb4Un(!f<9>D2 zv0Y*KoDyy`XaSX-7*Y?`FQ6$kdG(-4stM@r3ES zmzumlnhj+ydt%E3L%asZB{uw#T$~*f;(R4!LF7QdheTel#g0g7@|=|%E2yqPe+m8; zLv>8ox&LGe<6s~TH~HmZz?oBBHZX|ef(4Cm-+%o zJys!AqP?$bq}7_kZiQP#?I~6p_7klu33rRsLl^o{t^%6Ic9O zTU#5kJi^;o1|5&&yo5lW3q6iO)R$ABkcOC;W{&?mL0Dzf@x8;_Zpnv7pv-@kj^-u&qgSDQlL#_q>Q2l zxdI0R#KpuVu#IXP2TXP1MtC+f69$f}w322i<}Z>7(o04FE`i~(uxPjfnpf4bSB}-h zo|alyxkPnaJwf4A-F=_QbUjGCX7p%FJa1B`Tua?87}oj6p9HMup*|$1H#aT=U-Cay zsCy=Z;IBqP(PnMeK`fD4-G-2TxX^HUioVnlfYSmtGGm^a08#G7FWoE{boB?}h>S=$ zq@;A^Gl8aAA53vDrj{01WvSIHeZ9!lR4jcAnT-+V`9^_Z_O_=1VQy?*#3Cm73cV7T zNF`Kzr#bX$xvE(0c|+lZDR+gjy?NVxGa7fjJp);^GX_R8$$2T7;{ti?%!@C?Xoy@a9Z)wO zQdf@#ZWr{v7D(~0 z3bM_29qFM%gzQ@H6|4Jp?)QW5W)6Y0!2oxcf+YIQVr1NkjpKas3^^SfmGb0~q*B94 z%7b5)Uy87>{<9;a6z5vLOH9s+@_u|2@qqGuRTt|gqaOL2{KWYb=o+n`GX z`nMD3ZSOw%ql+Uq7Ey0gz@C1l?fu5uykW%JTc?!FFxzV5(o8EMis9yf$(hvD7*1&< zM)2_UAJ;A1jh*PDBS}z$seuc=FPb`k#;}Z?uEtQz`0Rj>OFKMXL_5DH|8YdJoD|^K zd|@iQaJag2Pai448^pI!cp@a_qzexQ8|Ty*$RB2-g;lP@7Iu9d;}tJ$Z}sA6XunL$ zn?F!v5>lCv?!osOOE#R)ITJgk8%NBV$pt zenQ-i7}?x<|1dS@+dze2(2}F&+x4}R+qczuc)=JP>iz7v1Vlm_y^lQ1S31LBH@qB5?>MjAX8zlZ$Dp5agOJ%?r1~ z&TI1fvJ=`u5R=srAm<}}q<7^&@{?Zls zlxQPJ{EtH5>X`2&@6UBMkAVDZuDDde@X(n(Frr=gM|Au}9eUn7M*l}=tJ2a#whYb( zNAOar$)O;77#Z*{$m*?+L@BC*j&#lxMZj6QM-gx)P6l;^0Tii;f`Gwh6?wU4FaMwwsrEjD^K8T_IW$Ez#)FYWUYxA0A6T zh(th%d6jwW-vtLD@qQtIj9rS)N5`3B7tU0C0i^Osg{v}4#64v->+Gt(yOTY?`s8!} zhAQU_+HSC!vj%f%iN}vd>BSn-K+nG&FlEB;4|W!8iB%T*oGR&?dt_rGY&TebV^sbn z+i8K+nL%V)&Ln&OzLHTRG6^4Wyx^&?l)Z+Hc~W9DdVwF-5r!NzR-eY!UWXK-X`KPC z$1>nVJfSget8Q&oc5hE_r1IHdZU@u~4;NBo-7r<2pujYU34uXV6(<6F3C zi1iOI8)A0TAzZ)BgPhX)&8oQFO`8Z_`w1GR{5m{fjyAZ%&(qa2z7)K{Ct$t$a*1~B zBHZxdqfhuiepb`uFU3=crqQ5{EN#Q7qWft=^2CS{m|nI7@aLQP`+v$9j>lgTJ6deS zR|Z?-;(&qyOu6knwm>!Xf7}`!uZ=wU6WZrOCTa37KESH^sLjqucJ0@rz2Wn{o}~vJ z7QC7G??xLH2cax{`IleP1C<6t-&L*vXMS%Xo6RDs^M z&Sa?7qV(CPrLf-6Zd40D0(3DE`DYr&`LB$4>+B|2)cbQc)gPGB)f^z)L6FWhQ8>sC zEfiM0IfUwY8m#Anw4$@Iz?FrN3~r=?d7?6|Q#RCTZ}xBH0m9A%R}W_26ZrTeSk`=@ zdIX6{CEHf-jWPN?X_loKaAWh;tC%4RfZKoLcP_gNTnu zg8aVts>7J;gCUr%b1eOln`B@L`0SL&oSBY}iG>VR#5RMlkQDCTC+K~g{g0!DpP}gw zD>8F_fav|Cc%+O{+3?x+=(41I;iFV1iP47-gY@4$VnM_Ew^7IfwBI-UDE)2Yt;Dj$=mGFR>X!FZ|SsCb}y60*-YzeOh!`Vu8IR2h&s8DSubYZ&>0Lxu;t>&xmD1>Cc9wOV1T3tv*Y%c;3Es5g=1+eD zn-{wn?t;m7h6ce^>>maa(2V$ChrXA46ru%(xbsziTLT+S@F>)C6OS{{t>#6?|Ddtw znTtnwb)*=0RcViWYO6Nh5#!gSB(v_lpk!rz_L?|-!PWkLJnBcZ(P=1L;R)<-MJ8&n zr@EVs&^B<%e32!eZ4$)xo9H<#L*m|aHj9p7=H|CGacL~b!7ULq&Nc5FWK{6sFxo9c z!rt^XefnqI!%h0uk?(jJ9&xF!Ceehl-N@o(vy~besg-6qor}`lXn9qW+p`Z{or==3 zaf5%VCaLDo;yf0nyvlO&Z9^G%M3d4aIF~VbnT__AHaIuOY3?=!8bEZ=?)=@fFSCi@ zZi&T$=->^zH*>5n!zFM6z7h0BF6=S@G$5P(BQ6DboiKSc{(GyqAIKexnvwo=8Qj-c z0siD6I>AZ=vP3Eo`2*6R1yzgVS8*6uY}bQ06k>4WM(3X!EpQ`ednI znjGT(ksd8S>o|UkfMuerDK|68N{M0QC0{LQ{{dZgx%%&btCqjhHgBR)EQ1JBAIvX? zaned_*!QYwkn7UT({BN|7QkMRnhJ-;_ODch#Pj=Hus$8w_v}M_Q_2Fr{Q+#)UCQ2nt19lFNr+h5U1 zIm}aZB~O6Z&lZXBxm<}cE3y~ok0wn|Z_&3W#4#bCE0=CH1mC#-t$V?J?24nVg<49y zak9L9S`uWwear?k`N9#LNN*6r0Zad6yF3Xy7*-vA=kIJu5zGN@cf9oq^n12Zy~Jv; zVg!;7fF1M4YBU>$Oi)B>j(5gS)laVxjOjo7)M|mR+#WWA!xCY~_pu4VihSvQzH&zk~_lqW_VM4%!t#M$t z3N+LQ#B0;%vLGg5#jQ;5x*zkEGWOG^UV7=`BGg^7gdjKST~wi|Iz(NRqZzlL2XnVQ zFs;Hhvug})(U?lpJahZs5CMo4#ESxj+PlH@kcBS#3m3lu(P6JmRax z3B64XPI($R9kOo+%(IrisefG$^(2A=m2zO7knivKX;Rz;4fEb?yvDtDL-8EjqgfL) zzZdEMkx1|Gx1ctqYu?n*vRQav5AE*&%`0Jec|VHXTs_k&z)KhPXrJ5~cygo7y(xym zZ)b9KzPNkpn8uwnD$)0cD|Ow_ZeejEn!~f#_mo$e z5BN}|o{Wn|vvBVOO>zWmM3MO_#L!+p-;-Nm53djSr@F95;g1s1vKLLo^BV0Y6G3|3 z@wx*ajr;4YmnRV6td=IBTc_W>k00HkeY}hxU9l8(+_Nf3c5N&pb)4)-D!1y)aE(3cQKBC$YKUWLNpk+*m8Ai`Mye3a9XdGmVAji8%cc zJlDgi()`MT0hvyhP!xIgocS#lSH@`_?Y&mAc8>+WD*cmo50UVkw7m8EhvogNb!-}g zSBYtsLIQ)Ay=fNeF9t7;J>_gzt%tiI6M>7GJ;=Bh3HX;lfQMhc$-DArTOc%jO{wTQ z?wn|n!cIKqTa*nBD4=|SK>#nf%0vqS4|d5^K)|ymG=Bc)TQVR1`*-HYo6zrAN2j1bhQZD?9YhQ2e zr;jf-cdtBB$iNEsFg*HL#AC-2(i)UIY{hUjNC8U?eLzly_ZmlQ5+cUzIf$!BsBTni z@<2!_Q;CA>=PFZI*HA&SM_=a+qM?um@UkW~I-1 zz1?cimA*}+hHZ`*-EKF z!tQwp1p%NZgvQdP4onZp0z##f<1Kk}K>s4gj?x%l1BaJJxzJJU#Qf%X0w*&3A&MxK zFO{;^VC~M27^c6;V2x+#@d?;5?9Vzv!-1mjh+kl(qD1}m+ABhI`WvR60>!*4T=s1^ zP`Js!6M2ZI-e7Hs`NRDN@)L?Wid?9r)2n3{J(ce1=7V2o!&I23c?VrU@`qT1E*P4iZ>GOw^HamnbUk_kku($m;U98cj_3ZBEN%L34Red7*A8XV8fi|E>^HUqb zOCHb}PXIGY!)tP_K3`Rr)?e}*Cr95zY&a~1i)JlIlx#YRwJxoM-1@aG(tME#sGFiP z`#ZthBfr<2nYIJ*c>H$~DY}paQzk8KlAmfce(GM!S)rFj6DS?FpaZDBZ%s({&hBvF1NAAbWpMJ4nl7MUUPog)4$g51QOFu>+Fj54%5`| zu!#_VRNV%v@4>%|=ml|YiWvW3ZG3Lel{%O({yDWCniRwo4|aLz&$(e|1X~n$hjtT?{zR;KSSpQJ^~ z79qoXgkkC6XCcxFLaYj2YHxvWTzJCYL)X>FJar!B|3uv&)>>3PKiBdZnfo=DhpAZe z*zl;)w3lDcy=;|EG>m#aUt>=jx*+Lc&=gIZ``948fC!R9Ky%j|t zy&s-vn=4&%BK4jOxhCE^#PT%!0{E4bry?OAK~r_=_%*7RR_hbv<-vhkh$6RkBKnX9-D-lA7e%?8yxfg~kG^Nvg&--!v z=cP#o3me%q{|Pic13s^PXz~>PP#U+`QFzk7UV5yj0f5u>Uhk%>(%m6G9J8!nwF;72 z#rfC{0;+Vf*VPQr`-m!v3pt!m@)KZ!M1(tsE%e*VB0297!*0OooNF%DEzq4Wt7F}r z34Hhgy3o%8-wFwNB!k(bsDJ(92&OK~B#7|M+ZQn=nF$Nx2W;|g5_z$D(yj2O5%BjF z80yX~c@3-g7VGNIFwm#S8mv5~1-}E)iL(ZHegXeNK50qJ%*@CawVeI!^pF%X_0zG5 zpPoz0a^@1ufLKii*VcuAZ_Gfz^hgCAt&8b!IUz!?U?X4k*4EMUS$Ge-q9RealqA}Z zLSGy$IVLJ&(yF5}-j4kASV5>_%fG}q&5cSzN!gN}fmDPz=D)UAqpUnzT4p!QM-Z5F z2zI*fWnd0Qa53$L$=IbPNeKHf?w|W2csLvdaw5u0=}PJ3q{x9wi@EolnvyZ7KY}#p zTB>u-Up~ z|9rrTl>_6U^=gZWhR$zkyZz2Oi^;Gw6tHafNnr;Nljg$R-C!St3EIoAtv!?04*>P6 z)|ubvPVBOmeQje$TfU7~POG2QH6|Fpar{Df0E5b8q_7v5KT`eZH4+dbM{VBMXjNU> z49C}Qn{r9X<|2b)b`Y&;GozT*;%dG=QJp|M)1I8n+Sf3^Xv=4r3kC56m7XYq0E86* zm1v`*-Qg&corNO_0VcKEaOSf5w-VzdX5owr!~o*Xf7e3=F1oI$xZMP8=Y$q>JiA|Z zqqVlh)Aix6SopbU%XX|FpevRv!eJYBq}5K`9u4(yz!#zwPh>een_856t+$Q@MK%M> zWb?l^J>4}>!)yrke%UbUVh;9EaomWXA-4}jE4!9IicD=tLBj?nhQ%A_Ku^^xYkj5arY;ED61xj+Kk(6sPtXn=&P({B zK}DAPYgt2<|0#o8c+Vf3i=3K6EWGrL%ZAS!oZrQ{s`u=ra^&UtT=p%z#*T{*=n?Na zQ&Hx=OM9$y;ri3C&VfXjx8&l6iXV;Q#m*OGOqkI7@w%q$jYY3JArs&h^abcQGWF$> ziEnpZ?<+%97;1^E6&dka`LWbnPfwPe=A7dWXsf@6oCww}7hEuebU)oQ-v>K%h~)O$ zGw(BVfs7xU*Tre9X*cd)>^!#B6a2t`FVwr+ha4WYM|3o&{2E8fF{cnlpd=1}4On7v ztY)XYXo>fgqrH%0M*pLIFcQ=SkMTq7iGI)f&42!p^$yuDB%3wX;iGpX;r6Ube`~?o zyOl{`t*uU^jhEwQd&9pXJ_p45(I*MI_Ba_AX(;-@;P}+gyAGozG_)tps~`-d62!*us0f5r-|%!BKSk%Ukz-E7-w3ADa=A%!@{iDKN@bL-WM0nHnb@J z*cNlO*-X4!ayFd~aQrmczXEF%EZ3Z78zhkyfT$uT%`oO9GXDOhV9VDi+=C^|o8a;e zGkH zuTNYwd|eW1%l*;jNJ%`L44*F|YM%Nl6pfmbL3UQo?aiTV0kccpiplg{aW@=y!HJ@i zytSXf>8tjpoV!p+=cdk)SQ?*q<-%h_XI1S!(A5mM*V^#UKxy6sby0NQgU=%+!e2&h zGrmfDoFEZH0e0&m?}?W*>f7JW)*FMBGw%& znlPhCzRi(csUo)NUVXebFaptB?HT3ALvU+Br9Zy0G#7~GFTb11;J#5`2}rN4E8 zGPi)87xn|g-UX_hn(q;@A_TLi<`zIoqIzRGU_IMqXtv9jl}n$O)h^nSZ*m}V#?!rh zTpG$CH4Gk@4nT(~%a#k`sH_|IW5LHEmr(NUmF%RNVWiuS;eD-W(SKU%yAQ(t@rsoA zOT^OwHpa~O^)K0qnsP#xp>y*Fe*kV7s3^u~u$0CR!UQPhmIdqfO=u4fTVVILtXE)F z-K;%W+Tzj?oIAU`y1Mxs_W;swoBcqW>kR(@OZOTRzf1cZH{N9VP?B|uFcQOX3~HGQ zh@w7XPR4D*@;W63hN#y?eqe43p|V(W{eTpQ@(Pwz@IsTDK$y&f#E*lxgA;6D-jL$Y z0&%S@Z*EGFV0BwfdnLQWztiyT90^gNs(c#^6Z}QdBpc!6`ln!l5QY-&X%GeJ8Y!GL zhr=5R1@NOTb>}ioLUS;|4V91C1NFg!&!_FD%!|IGRY^ZOe7|CX>~E5+2dT`N030u$ zv9^X9pQn?lb>^MQ-#Tjq(GHrU{T^{vQ;%?3vU)Q^kz{r4P#&1L!TwMFI{&<_1~*FC z=2zv?PQrOr-uJQP{1U#ugpZ+7#Yy@-%`sYcTxUf~1UYs5`vkB$;N#unDDnAZFS{)w}&H=2$)~6@v=&<+CsV&K?;13NlP#4TQ*+Pcp#T9>7@i>QN z>M%dlqQeS%E-5nx2o%{d7btrD;{lfOs z?gRZDUBr2Nck`gfpDW*5X6PTSTKli1mY6Y9oB(`AKS@7yJ&Q-{Xmc$;OV{Lw=4g0S zK0zKalt)^9%$%=XUXJ{x<}-0sSmvsj=y>IL)UWJb^J&&wURm=oJ<$Kf`M-dhGlr?d z&=C|N;MM*UD{eQ7d_3U?QsM4Yg3x{YOru7q)j<>KXSK{X#P$jd6m63HW7Y2TFVa;u zA}Y{-$grnO#U^njIl?*fOED&FBL>Hwy`V5%w9tsU=Lp5e)|+KNq$7E20kE3eaMw*^mkoG3;$PG1=E`6(=y z2kxZM-MN|0t@>cKXcsII>5GQjF>m?9Gyuj6G2guPCI@~uc~E+i`I>E#Ny?!u`P#f5 za$Y0&qDM>Rkgr|(D4e4gY^Jn=L4>X+mu3LYr~pv-Sfpk!NE(jM zWvXZrR7DY25CL1=?>Bhzk)`6rY(eokA<38Yd}1>(1%;^raB;4~G<43FQ? ztb!o*!~rxR->rNVc>xz}=Y93#sO%PwmPZDc)TwZ+vCUbgyxT&>TGU# z4LMBiWwTtJXMYU%3hplbb}wU}&6@0lSi8GpuRWxxg@+$pWgCCv^5rqD2rQ(FG_sYd zkJg29@IQ~!GZQt%goA+B1;7Ikn9XJeZ1?{_YxIZ|PO*=bX%tIQiut@oWfP0nVB^bO3I(*ve;$4>&67n@+k50= zl#Nj%B`jMr(N2aR3$9IsR*Ykw9_cgV(wbl`lFn$|>v8#APRlz&AtzP)=Zqqbadr6D zOq9TeD>+(@u&LS?@e>$81EVetLMh|+3TCBCQ0@x4KC`%|837q7?U!K_wxG+5MS6#) z4dgAi?%?~HnHNo~7}$YeLZpRKK5K@jrJAjd;uB3e?xEfBfg1Un7*2R8x7rv&oOkq+ z5`{)(n$?mdySWjr6H}u6NHV%*DKq~Vvn1$MW-XdKNdgh0gIf0v_aJmoGD{2FLB zfQkX=n13Ms>q7Vkc8cjar;0kGLPo*Ag28-5gDVb1wu|O_glP$GKciQo+ZEZJ!^-j5 z3-Y~G^i}(^hQjO)YFNoI7v@^n)njc>)(AMOO9Ovoi4{n3*@W4uCySeK8BG3~wGe;> zDsCI^{Py0~_+4wl+1IwBPB!#}>%gsFO5s01Aa-F+P*r$6#JbkPKuP~NRYeULQQ4{g z4#}1}=1Y}$JGAY}v^uJ)KAGTQCPd{{C3a@74K$??D#I_EZGY3W&)}PoJFhRz;wk|> z{Cu@tn6hA>ty!G^<4tdQ<0Mi5|FF2WJ5^abF~r&^_hh{QdACs9c&f6>xrKvf-K|qx zUkZ7t0@*&cU+;xD1(UeHb`QvUzBv$pZOq0LSZ`@z;;B-YPzi1)gW8)f^*Wgoc>Mhb z{(~nDLGG=Yjy}4zjAh<3ID&fHYY!#tx`;`A$^f>aA(Z$J%=N*e%agZ1NX_sc@Eng@ z(j)pksH`i#^<<$}bpVu*T4dJ6lC);){|xTb?&?^2v2IwPFqz;}AQlQinoM6*sh*fk zQ<3aepDZUZPZ6v1;A*4(dPu{4UC3MQj)nG7mE@>Kx&6|n3uSp5(kdVAF9gt_W?z^o zZMoR5llZ98`U&i@(Eg}e&+5?qZ2R8(_`r*wKmCgAhW|K{w1IlH>5P!{Gc3{zirYd8Pgc;c($ z)>*-~xzIXU%D1VW<#3uuqyO2m98r{6(yD`c^s){v8sG&{jB*IgDho;2f!F zoX9 znzQu|4v;D&m}*`wmf7lrW3R0Vj6SK3>SIIvel&X>0mDk4C)!zWc-me{9pxndhXPYN zyRWFp{uWbN#JJZsuCYB}fX8uEdxI$jj#JGRhcg4!9MyfqhA>o|SQRsi0?qoEMu@^j zY*SqE{zc{HTLKt_F3oBqnYu@rzd2Z8^07|3o6u!Qq_PKmy)vYs0k3`nLG6#tp`q{Z zOLkxPw(wtF4f%Ou{d!Y7maJboNzx1>oJ?H2b#CEtCHf7k|Nzc`I@V`wS z{;m4CAlZpWxfxyTEPFGA8fjd0iP03bSGB)iuQnr@Z*L;REva2~gZJ0o@&?hBXoAJ4 zk0pPTkmv5B=*4Z`YKuj_tMNRxV==kvj6;_^9#cSImyP=i23Imnz8^N)7e1M#cxHym z&CVDMRNatiqNz`ZCD~HUOw-NXmjSr#5ANWBUwxBAiLNc$L~Dr0&PJC$sL?j!aq_Dh zHT=A+0(~!dLibcK-Fx?&%a>>S*9Jb1xr4yRn`IFP30p|azUUVYx^n2Ynm{2G!7za^ zmY1RFs>=-HHUKr#&H)tjRTz&@EJABnDA=D8Mof9NSwe*R&i*|cm!w^43eH%p?g2?2?L;b%a8F$8{1qX z;_&iSYm|1+v^(4ISZz2Z*CWeh7<~cz(%*|G&-IJsFl~a)o{o;yi@cuq|LSf}<`q_* zUhqxYR|lCNvx57Cyq2y0+S#>Va%_4$cnwJqOOvy8aS=l2JXABA*`6ga;ST_i+Gdn+ zBc1eSWxi6T?oE9o@MMS`T(h+Q+CS{W#67Uu;z2}ogz!uXZRIQ_^_f}9G;<2+_$tM4 zaGII-BlWH$pt*#bX?LnBZGe){D{xF3nQl2qnPm@h29ny{378S#beR7 z;cXj=EG;yig?kz_pv;gh5a%bzkx%cy6Jjmc+2>IwZpc9$m#hQFPEYO*6lhX=-;0 zOQqGA_T$~gr}d3U_NWbQSgoG_BJ63s@|&}5oo*(e3e26)QG$B>2gS2t zn$Bfq^Hi6Pt;&WsMlr;)xbvx-Km&ML^qze9_6aPasL&!I;QVbh<#kT!Ng?9m-PxH>TPXnr7azCdnn4=KbSmd2!1r z;*1wNZ@zaLJxL2_bMz4dDH7T(YJK(sdq70I(!f&ng{UL~WB7X|1ZM2D$U>9j(K3aH zxJL^2(_u{_^{QB&U#xF@5g_&|V)?Mz?uY9pa%6s+*e)P)j}kssrqc0&(Spc;$GDL^DYjAqcbK1ZG_j<5ao1WX?Wk61j-p0v@4T}N<`EAcGMP&Yp9eBY zQQOJ?^tW@9*CH1cZg?HeY~pr$=!hEdvhtur_anS=;Im9piNyrGt%nV!;qkzDi7-Cn zVunW3R zBLjW=_)Bt76M@#0I~bj#>5Hp>Fb>s%nz)0$Eksih=7YbT`*)R>H{5N*Zi7I%)+ZNQ zkH{&Yk~+mz>NvY7E*f`N0)lPGdsbjuBL{*iRn`qefA>KDb&fkCsxCNG!X5<7o=Lq&M%k7NraAZ#v5&p zmd3#Z59s$8ml@92xve z9T7dJ2{wYrMZ08K$H^FjpLbRbDHiN)tTF^=t;>g0pcBrK9g5%rQ|}e36{x&SP;r9| zCv5Nu`RPL;W-3-cf`|Q(oI#5NH)$rzA4H1gyI=BtNo`$j{Fs^|IC#?#U9*B2r=yJ4 zg6#-GYt~pHrw><;`+pA7X|6RZ2%1KGy@LZ=HOV1!aeE{LA9&nvWI`Ctst(&y@$v*%9x(&KiGPN& zz`+6oxp0dWujdj(i?Agkb;~!uK4D#YVHg;n%kMwGogW?TcqFPZ_7DiL2Tn9(85LtV zj#61j5_Sy)wVAPhiwbxi7QqF}*jHmlb3Pq*Np3Ju%T71VdA`~`@JU2ELik4|@+R}f zF!VR6NeF=EwOK)-JZ)1tGMXZ?MuiAkOX*=22MiA3ka7{PhMt#WRjc6N-8A6+KnvD! zG{P3j|5kVZatfMa>ooOs)0 zNLtHRE)qZ#V}|8G6sx2oiYTM-R^5=UK8%P=+-W?LS*%E@2YdNARaq3xgpqU{VytNC zuY^wNs&X;^O7FxhEL=E}4U1FQo+uNzbzExbiI+9){oak`+ zy2}Uz#;Ygqc4LU*B_^=g1;#zNi9~h8f=AyJG=wJzxSv{vK(t5qX?3R4e-9BIW=@B= z+&WB?By8`9*tqXC2q|`KPH%8Tal;~mYv5n9vJ4i_L^nXaOWf12=-)+I8Q(S;tJ%BI zBVbZzh2;g9(H-@(AVfQ>YkVV-^(o4V6ErJhisN|*U{=UlKZSGcWJ((Qo-_2FWF%HsaEpG-=UruKQ62eY7&b`>)YIF_<{y!NJlRT^qPoO;~Hmo8efZdrM zB^I1(|0Uj2Z_gP(28s=_m*EZLE{pyb|DhGh3oArt75DTtk&o6jV(I~Ww7X)2MDoNw=0ZSm%T+PmTN4)FDbi-%$(}m87=zchvu^awuGbW$er~CDH3&Hl! z?&kib$|wKpgXTcfaoggXcW+f3u(Td{`J%u(sVjb+3QunYC?v_+wRJUNf>Q6amsY0R zppN*Oe61;+if?Egq5WSwp=(eQel9r)@!zqh>slbvL-iw*Q(~+>r)$=bAC;yEY6ysF z(k&Qx2FN#<%13&h2yKW96pwTAWwDUj9MRkWqyU=7#-gz(mrR-XFpZKV{si^QTj^yR zv1~3w%=v5fP5BeQ3pnmGGzrYJN`rlBdy;pJd%g1JtV}yp2O%D<&&D*a+47YYCmc9m z#1nKkuiEI23yhImdG-OlL)QJ}-7hZtJsN+n96a1h#W)9sSWTlO9I<{Qr^KQmdgzbP zpR`GLz4Bk(S3uT_3xcjBj9iT`fMPCt-eZKcyjsXarhA8LEqBUpW$J;7FR zlhe)*S^#T<-Gl6w_R#4^+{|bZp`lIjWu;~KyKY(+2P=&-BTN$FBvRPWPaPc@v?OCj z5$1QM25G%cw5)4>)`XK(j(?vy(+H2WExO!QU?36by6|+pt}*|owzGGVh}PTCeqp;{ z2OU>b*-Y0(nJ3etP|DU+t8?&oOiK623t4J|(=@9XpMxP)(4sGn9hun{S{HaA^??ga zQnY}TR?*CmH=S?e+VqM>CJR*$)g*S%P8CD-m||?tP2OhDOXRvBbGAPBR=ysB?PRki zl)=zQQ$lYlZBl+$k&xaBZ%VG**R|9m9IeGy20{#~e_<2&h-7EKSPEoNV2ZC4cCX!k z+sN|#jZmaD1_6P5io&|Mo^s7mI^0X$)BhyhYhUwwD!nFh7&WbQs3Nuad-M$*V+H`0 zgN6ndpu=tTFw3+yMA!wigG)L#(lB4A8?Ck>mI!?rAM;mZx~5!#bWBVfe;Vw=ccmbj z^3>WYH}l{|cjSQ|Q6dy6$m@e4P$4H3-dUqo%p;e(+|9};-)yZmG$P~R@U1qSwnr&v z$8dM6vZkiCMl<}QTUX<_uPebQD{hYIUJj>&ha(4!TU7n#j9A3dT`#_wOp^#Lh_h9Z z1zAy%I6A2znm!1`hhIzhE36Ria*XZF933;2<<)lHmM0zkCzqDH*zslBhjn5SH>2wc z6gcWekH|ZS)dWI&0mC)-l;f2FzHR=_s>B84LH*JiPnw(Kcjo|+ zjgkgnv$7VCudl*}lMv-5S&W)jOC~FRM&0D_=)lLOyZ^57%mYmIeAWG%(a;dF7owO~ z8^t9~F_||uvSu^H<*kO-#>=z7A2>?0lj}7ms#BR4&Lp|ZW=~b8nrlOV3>MMSz?*L; zRn<#Vvu+NKQJeddm?KCd`snc1X96*_szO}rvQkt<*?LkNb!5czD!^@rLh7r`t3w{N z+ptZ43M5;BD3$E6nxeXvXw>_bc#<`$URX_L%-&fweKnZVNBP$~u?r);C^T1estS{K zOHv;t#K2~L6Dj|$8Y7oYPK(5wOis;>%q&VaH|Z;oN^gYJetYN6Vjar3U&aJ=CdIFi z<1mUAE|mC6kI}#YA2ro0tL(cFC!0hc{4qt97(S*pycbQF!~^jB$L@(d4%3CKN_(yx zKP~Fs`}OGHttI#`~~Xtm~5>o zWZ+?VfE#XE#`hBmzpWqA&n1MKOJ>7{ZySQLLTaM>_)sdZxereI&4&5^UDv_8hJpx< zluo0>_*X)||7M&+mte6US_$X6W({9ujfZmBRk-;}x62Ti2g9bM%<#48z4Ozd$3Za1 z$$d3OlCpW%X!Rs6ljML_dR_>*%z2MxA=$LyNY9A#fBd$J0FA;!TdAlZo!?$_{5!;e z4V9*@veU{O4*xQ#Sy3$`7pXy&lamsarHS)!U3sm68c%>T?nel^$47@b69U{wdw8Pz(X;9IFhksh2A<`l&&R7nF5T0^t?c`7D!8~c3keWT zMaI99=)C+}!}y?^J0Gl&p*!+lxjgd;c>3$dDe2A@EF~4RIdcC5HZC}2Zc6zC$K@qM zw9vu?xR4qdZ^9?i4bib6?DL`xNhCB0&a8?OfiB9rL`OJ#_WCXj{r%lLLKSl%aoWnR zfWfPu14Q{M@RHhM_?PR?O6g1888iFQjU z9sLspD@|6o2=l~Bu6=_SWZg*xQTNYT)72hH_~T0!g>rIxfiQ_9>o}fLH|dJ`3Nj#4 znKO;eWCW|M39C1)psz8PpXSqWZ1kUs-Al9U{@&f;;jQoPMET*3pAOh8akaGDGNQj{ z;6sww6b`Zp8K0GMZJ5%B;8QIp>ORlG5kf-BW8@97PP+4g^L}&o?Wqi`FRChpKtqddvm$;+?`xI%?hh#MUnXVvVJ^q z_Ve>0{%)@Px}D`AecJ8paWZPJMdi68QlE4G!H@5~g+0>r1DSY{LKCh&dS12Fo`NKj zIvu}lPB-Q_WV1J`B|`)IyMt;ReMQ_X2&3qo^2tE6>QJNcSQ;=0l3K1&`lF4% zL`<>s1yk+TJqtnDt(@3Fu=;XWEZMc+`RF9dqilVfde!Vx(sDdZ-8^FC&}U?k!oSfR z$w-?XgG$TZzGj}vvhb^ZV!h9%{u+uy8TU|yGja3Bkvzu1k_(!!DHRl9V;p>@1k&`w zEwco1ut6o5_;fOxe{&4O(6)e-oSjF2mS_XcSL|WGakE@U@-5Ss<*HYNvQ|dt`3m}Z zzns0DxL@z&`E$KI{IP8$#((jCe6P3vWNyqzNLM;gE>tLXdAYniI9WPq|9Z)fy#uML zc$RjB6k-ea0I~rIjYJuj;I9#@@^&1EaBj-1Lb>$C1oGE>Glx9fy_n7Q&JX1S#Z_%F zW5z?q%Wsk0{%vYd&$4(6vUWR6Y(IA?h^F{h$?Y?T4~4ey45Nlm+4o&zS+#)>`VN&w`pxri+#D@6X@@ z(SJVSGr4r{ub2PrZ1wi``n=hDyVq}j_QuZ{zxJ0Qqic^3WaYb@-wO=+BZRM&{~m*Z#0Yx*^{Q!dTu#LJ>01 zT^9CU;ArW<_K6_z7z01e`v4BOAH00iJD=^Vl{Y)@K7^2; zm4Ew1#CFcb9^pX`W#cw#fC|b*KC46l_hmiTW?I;+>N-9=)^J=^Jw5bCuPJvgP5f_^ zlmp9P9Z6141RoSq8Mtp6bR41c+@Mt%+Ry{W4)#^wvUH5T$8GjweJ(>fKbzVyIG z4W9Djc*gO5PEnUZwu1v@I9sF^tCC9|?rsqigUIDSzewC9>Sp9Za;N`ls>q#1M~p2O z`T>1;unI*$n)8uA8qU=IRw6V7InoMp6zNIM-4m2< zt&TS-F&N_H^Ic;%_3XmG<|U%2-MS&nyoS}{rc@fpn0GO->|u*Bz$5>o8(+0BBaJolH*3sp`4eOfEfhQ1adL@0{L0 zlo3#zwCeVsub|o;nsFE7ZPkkC@Mg!)yMOl6g)twgvILs>_q}ughp!A{GOBiR8w|ZE z&g%~>O`^2c*3jPp0L}p~ALu7b^Q%=<6ymc=4@-5DehsWU?V(V6HBjv3v@NbA<#mah zk%V-8ylI=r`l-3xoNQ80A@M3(>FAmcJ_T_f{oSX7<|6 zMyF+ue*#sOe&u`rqoeo~#iAi-=y|gzYC*W|U#ffZzdCXT-umbHz9TJxRi~UhGG>s_ zc3gu`Z&orP@)l?kK#COWdx2+4M}v+r+DF;qff*t}f>_=&U9&rkL-jt9vGif(1LeZRgwzVmN$cSk;d@F&Vg zatS)f5ZgEBWu6?WoR{SzvNJ);j4Amb%1>0f(h<>tD3M5RO zr!+gGYKKoIG^|A(Ebx5Gj%^!GYjQr_=O+akEu}9$VDYOclLY8NPyVd)T-7$94+P*R zgKXz@VPIa^#uGdMkf;0>R0*0vcM4IyRvqbVgjoN+7n9zXj?~cp32VsHEo9hMOAoL+ zdQ_ahp}xf)5-6U$&NNV{<4-U*Mnz6axdY6SpAgignJVv*LZEtc) zSS58r-EDHr8ku&E1Q8$~+n{-WFF!eCkh<&ucG<-%Hj~8Zp`mKZH`Pwu4=jgt1>)MZ zIIDb#AVVxNk*AmVziqp1-Qm^rDEL8-#q9K|I_hu0^jtDIf^9P)c*o8vG6h6;+Z)L5G}}e3ASN@QjxVMmN5;fMzyBXXzHya z+AxYn$6cA695c(M=1BT_S4rULAtO*Xc8tNj$ncjIrnkYng2(xviN4Nn~FPWGqQckvsn#4RnB`Zs;Wfa4cWZGg4kdQQk*wO2CmO{!K`>`|5O+BFC6Wq7os-c%1AP@$T34g86|4T?A7gc}+x<6cV>O zYe5gX*SAm#noI6M<#j4u(UOQVQkS-tH3}iz3mX2Kg*2JC8xs4^#qCRF6l7)dTIC>c z=I8R-&^E-+Cxk*CXQ#O?4Hz&wjc`Uf|Jh`g(JWG3`WNmyCb}kQ2)1%a zk8vo~l%Y!drjlJ2Vsn+kS@-C+u=t--Wu-3Sl55%(pEx;MQ0FtW?~pk&RQe z&ItRuFgyg5Ik2&uF4&_}u4~Mg)gTBJ+>GHcb=g0qbL3C2+d{VX7;k=7Y+fAZz@b(`nq?#MQHA1!{R1C}>zs+aRgKpZ};s zIz1|Wf#izDr8Porw>RG`c_qQo(J<|oX<3KOrH51>QeD;?WJF&s5=&dQ>?SO^$>-$S zPYzdv>uI4X2Oug)=%UkK!u1)4MV1nUw1<0RJl1fNiRyr3b_QotiwQ>rKOl7?TVelG z9%zb2=RiGMOSp%-6iez3q4_?@V@dJe~{8b@>wqnv(3vg$$#!_grnfH*>G7j5k(wOc=+ z?)9_?)N=kUMiY(na_Ku5G=U`tn8pUaEq4fnSUir0+O)8TvK(b)DCFnScPq+)#+=F_ zz|h8z%)Degmz#Hu+Zv&qPIK0{l$iK$^DOO01o%NZ1@Fty@|0#WZ2rYCio z1qrIj@LLxokr9n(qgo--b5Cnb*Roc(+-DE`k~3)8qO=Qiu>A=9+O`SHHp_t48nOSkyMVq|6q?Wm*r56PVLdFLE=RQ=k6E zIDkjK2oU})#8efF(a56CnOlv3tq;QJmE*w~>FP=F0_+}#KDz#eP64DAf|5%T(!KCt zVO6D!N6cE!P7<76Cl6Q@OB0M`E`thEdcpJs8{gmLk8%?7slV@fmzt>HYAY;t_pz`& zH*GM~P@Uz_y%>G>I^1MZaW2~ik}9h8Z-SGJRxTJ1jdaa&!gl^p?W3xXzr_J)vNOI5 z$w?2@NE)$pA9pgkVLg)9tN{*IBoiGXI$$jZ*&RpgRzb_DdMw@~f2hwaOlWaA<8P?> zO6oB70gfR{)-`1fLmo2i!30c0Cdl-NpWh_j$rJ|0fycAACTe3)LSRERBQ%X@hyg0{ zB(9q5^}<372{GJ^s=&^F?wIIzR?|_72oXea{IXBPgyCf5{W&&5VF`UHkIsayid_1K zcsdF}sv_qg0o&1$EE5Wa9vXKFFUAS%O12KBbXAlW4}sJ%-e#rx6Mes>`r8==wk1{b z1~5m$yR5LAX{aX2TPcN=?Vptb#k+M9n+RLz6sU!kqPJK%e_>Z$VVI zyGV%~hmLr$!jG2#va)pvTJFfzp!9XfteiDH8F|Nki3psQ5=0f>I{i*uBlF>?#)V8K zS^dSQZ*l8OLuM5epfybIqXyXJ?XYRI%D^_GHpW3%gFYG5zp3=P&9dBS-IoEZ7uTZS z@EF3hR16X_eZeq|t1gkhCuqEs^HGvNa0_OpOjW=*zW#VCfd%)Z5YKrOn)u=1DN55q z!oRHEHiPVSSh~iCndm?xv$ABDtq1ZK;BeLx2qFa`vu+kdGxi^4X!dYU--B=tgD*MW zl(HcjVZO^z8}PvCp*`=GG80HrF3)gZK{0~4#5n}Oh}dMy+o&UXOthN)PS1<+f@~8Q z6r7@Fu_QEZi{<0LBbHF6!oF(_TBX$;MvSf+S63ZqsqQYW1FCKpY zCFE3syBguCRm`6Y@L2?LDx=YDr3O)|T;t35CvD4!^>N;su8Bi0;;7PL`kSd|cj}4# zd9z>F|3FP^Bj!AKLpPWd@s$|EtUw-_S(Af+xUbZi9!-{ZN-<;vRfuuB^{ z0g*U4Go0wfvLYUpttuKFYE0q>m$U6C)wB~u(59tx!!gJ%lAAtR&W=YwsE0#;VKA4i z17|E@9HEop%Jl1=&zKyl5(c|XSiv%F0qagEZZVR50_3wc_pGrarU{QyIA$B`1;8EP z8c_=xiHsMoG{()DT1G4 z9q)l!xZhArkX_~$7C1}O9Ru9`tR`BQRZh%r^Kee1J0W`<%X88oDX__;0csykaG`!*%+^Xa{TEMaJz_UOS^x?;&sz(sZ%59J1K4Xhnmy(9mgcyzo{LJFLJ9BJP&1 zPO<|$@bFKvzYP$3viq=)ttZ`IzBRV+4=#?3=e>ETO=9A_!bqXwhq}BxXcP9Hu@pGoz$I0J@WZuuEb^eOE&&??q zmDq6`ccc(}e3N`+wdD*+T~X9}eN$nU#6=LQopULgH{a)y^%sYSeCgI4`w_wOXczwZ z+;gu!*#&!hj-z;#av0yq7veir2d_lKXi@Ts>WX*QN!>J<+E1mwET7n2?0^AEZ-5Lp zdTF(Fzw9mZF)QCIE9BH~42HKL*qqi5pN9TiUP8_>tt7$?V}QY;MHaM&U39K)1;mRk zV#-K+2;zAK{Cka<2 z5R}I=k9FI=Iqh(BT_&gJYW!XqdsVpewdh}<60?YM(4#gnF*LVF(ynNdlwoC&*Nja_ z&`T06XNfigYHJ8syI*aMjlJs<#m0_}L1+`l#**8h7QFiXLAAU!umxD&K*YjbaBwIk zKXJ+g3Atc{)_)akVhtWus-m!_#gtInAyNXNGzq-RZQE&or~ztb%J5>dmC?p~sc?nI z6SOsPWB(vhlL||0okcE*8~O#eEf)R(MU%$l#{F7T()1Jtnwgtony!oIKF)r``4Fd8 z(S#o!t{Ng-Q2Q!vI@6|Q9)|p)Z9#=XoWec4q!XUKO;Nq<&_&Ykil%#SuS3X@4AWva z$u4wr9in-FV`s_P;NJ8#n`{>0qh1C{n~0aUWtzzSY2j=dtOb!4nRZT-_cjg@ugb;H zF8e|w*L*Qv4OtGFmVf6=$mQFXw1m05!$|pzAGw1aShqMui%#p5*0=%5e959JWX^2E zR_)>)i~Z8ZA(B>}pYmsi1d&2YDNcu4fplgIUd zyDGk$&R%+)HyQ050qNcrq*S55zpe&pD{l0O$O93_|CV%U^2j_M-krVXAZQ@b(?}Hm zureAca2mv5xlD-aSX$;sf`i~(J~oRl0Jw@LRIW0P{6+vz^HM*e*1A*Ez{RD5jfah1 zZ_9XCB_zEaPX`tg-}rXWi`S4wMsSpk3EW$`#F7H%Ma~;$xdwK=pefgustS!hK2+K|aC*c|iJgAB>JfmStO{E*Tf z!d>r0{}1Q{IHM$mlOcTO z^A|%s#XzQC&Xud!SSCSbAjK5VFmU=VXMgbct)q5(vLVM`4ROoo+jkKq9TS$zv_*tj zf_cjvgAVvfG9>J!MlG%F8Uw^(CWzGn1DsRPF9iE=MfvY6HRb^SV&lXfW5VOh_ga>} zlu@tcEmx~pMla%mAtVEXg?V7MxoODMW^%SPK6(J z9co}7v6!K-749rVMNR9G=6KU*I<2F2`^fJ^3K;v@6Oex@%SZ8Aj`+I1;NtY=CV=cr z28{WrE=fw}evPu6MyGnOf0N*1!*}K$*p|2RJamkX?vUTP5a+_>^oGNG`feed1h&DN z8^)oP!o3H@OoU_S`Y4MWE^B(733xEiNL0|c5=&`J`JAMqn^o6_v*+ALZ{=5VPj_orG8eh!d#tiu>|M8ueuUBk&x|NR&dN<-iFfl1InwXSa z?i~9#gJECr#f1C{WX}{Mn|07D)xIu6u2P$8 zc#HO~#N#~v<~yJAOG5c5kwi%m>y8~kuY$bZFuc!i_1)nxt{*?WpKU7mIFUNaMRCD? zah|5?eM^;RQ|q#*G*s2xd>heO5y7nByFcEEMY?xqbU0%dwE(fxU&~#Hi@sO!$LN`w ztjgz>H4sgvqCytJ#x!m1ihGTeuijd)2B?+Y+)%fV5gfgk&L(t3*0+&Tk($TG0_hdX zCU&c)$g&;6dzk8Gm~iaQ15f&Cca8)T{PMXRf!ai;nPdTnK#urlAvj&5R0 z{Flag6Ehr>4t}~l1{o+x1V&aTPX7;!qm$J?+|jSPj-{WXaI8(Gf`_AF*TAuqEhJD$ zTl+P>V?Xevn=ds|O1H zKkSXCsJ6j_mCqZx^^cWu2>qPzR^MO`WO&{^tdFf2BPO?T5&w(cRMx%Q$b@^VdiY-_ zd+lhorZZ}zhwtG(ji_Tz!$RU=Vj`tvcX(M~+j#O``Z~~h6I{|w(`?RkGu1n-x~5gf zFHUhs6F0)`gGdvvgMBgEJvsK^hgk>}Ivreu4Q9o19N8;0{(r%n4LSRWm{|FpbKu>^ zuocp1Ztl5`%w*Px=4%?QbtBz*C7#$)KVH zXBCC;(ZgM*tRz(5uF_Xl$V0H0soFEF3OZ;i%d73GwOBOvK9QNV%n@MSk!xy#F01R7 z`rfFIFs#rJN|#x_l66MEx1uWRjSky9JJ)z25z=_)Y;9fkWB>k}A%gbpbLYvdL`b6c zYE~|e_-vm97vl423LhiK@ILgkDa~Sevniu zO)KB8A%M;*PB}B?BTP#e?u;zUT>cw>2FFOw>l{8&Z$k1SDJH#;DP0Lkn}+L$dTP~6 z*;D}?uiB?3Gvv{Je@`zZKXkKx;M`}5`x&tC9zw6Fd;G(Yfh?4r&L*4MQ_=ud>k;7V!ld0f|G3&QvEkF-Q$njNd6Swj4{DQq z{w@HTX7cFn%zIl=_DH)mX4bb6nyk&Q`Zpid)M441rHI??heRf-9Y?mT+)qL8zze8g znB3H5%JPyqT@7bj5k_2C%+=*hwr0pL6KI{1W_lKStui|s9TsOD@Ai_)W-SFNpRfd< zTNY`)M~t4gq(>EV%BXEra}Z!hJ8pwo8cx}i&Fy?=6O?`@ycVIvPz;x~Y3n^3nV1Z^ zJB4x%XwuA_8l>42`@f6PKt4IW?y`;aef4A;`pIh{_INV*g<|WwmXt?I7|e^?`leWf z^t&z$etcZ{K@)rH^-{^a9D&N)(NcMG(s+bQ z7v|HUNS3)1u3*~kr}~=zlG8W&sn4aW7U##EkHKvP+t00^Cq|co_+c?77g6m|6A_+J zBGS0!H}di6J{1Za8L~#9GA&<;&;uAWN$tPEt#k#IPi~ ziu6{9LLrgNn^^qwTc{(9purS0OC1xsnH!4nsr*%Ing&loXOy8(OJ_Kdb5+&C`JhW4 zd%OmXrSK>hwvfLFwkcvrW64>aYeQrqvt8h`g=5TGE>2@PGeS2fQ->qLpQzKki(xz! zK>pbQj*hLP+bie|i2l499X1@HZTh|v@XXZzpp&7EMll4Jojmdsid5!iK0@0W4sExH z_9O2c-e%#%9f3WaKw=r=sLQ|;egHbs5Sz1NSYZymaAU*HtPvYU#cK&&>?NG-nC!EU z?nfUzfpHs7thI(%)2JgG6LK@Ba`XD<7LtMBMvD82ty!LMfedTp+v*&LqJrQ@y?>6s zfNy_lWxr|L34R@bth`p5?9Lr5k)u<_Lq8<1%9j;R`q*p#KEpnq z%0buZ6llj4Yg}D3gs&;ily+Bf$UhI;R0n=nj68 zPHvlxGj^S+LoaKy$fH-NEIL$rdL2Cr*O1R^3)Hw|JPlL8Xx@KQP9Vba>(V>jl<%nZ zn4H>Qee;6lk1Y(RXQLbv=KJisSP-pqB?2|oYA1={F2?8_fFM(vAz_@6T)#GSM?4J; z>wwFs&B20>923%3591K-qKHM9hVeazj>EK6#tzdu4+B6KBsK&8D{qt;$@rF@J}%-E4fl&{u=q$x0-)b|v&8 z%NJO*@1)gI?s?`;xjEZW>#LnFs=LbSx)LVLM$A7x zXjv_b7Ig3PKdFE2SzJ!sBs-|ZPmouJw~v>>%ZI zr}C!1+uPgw`QJ|N@UYkClfZwdnk*hUQxXVkzs1U>SJ+Q_eB=DK@C<&+&ws)GvI;kX zYV#M9Klf)v4G=KuLHn&CNYXD>yf#s!2FTkF!e$_ zk{NI{84Rs_&&1zNO#%Z+&wb-|HHF>d6svcsBh&%pYQv5 zh~;chv$A+Wp4455_{*lU^RJmYV#~U&TbpV8A99Y4@@p6LJ7@ZiuzEM}&$kIp|7B-1 zI~6YH3`cQPe*^~jB^PPXQtOVhCaBx4yvF}5<~hwjs*Mp#U=&L9P~ zVD> z!bfmDsb)pU!Sgl?HNzUpujF6`R=WwMbrI8Ix8ItOPP*R9?I{2Djip04x=p>AxQip1 zLa1xF#(5SZw{M^Cv0tka&O9$QExQ1r#|aivVJY@?py zCa?^gwd@LKGCoiwg3q^Ya5h<&9?$xEj$KV1#GoY0$ojRBVzi=7bW(*x^apqm;xb6o zcAm84SUAOJQS${^r4H9mc=j{cjxev~DV-=i z!OM616<&$sKat&dT|dgdj&oAv>=6rPir-q!RJ}4o#T~&sDj9PxehPKIVFTu0%fOn(OxL5eC;DOixHOrQ zTq!3>9M?1iKUNtkXe8-Fa~}p)34h@~EkJby9ar&(iS@{yo3BaM)Z#YS%HgQ#vKF+o zT$FF`^9O>jFYVzYM4n$-xallMS?%Zhme0rB?1xdg!RME8E=0){CrxJ8hCWQ21K&Gx zx2Lg!%A4*Xl(Ch-`NZNlK`3e?x;Lz_ca-xN*GH2f!wA`27srSQ7%riJ*1m z6qPCx(*xkWhtunB(#k`4BqxW6d;{JAx{dm|c?F}b#BlLWvFqf8{domM>wt}6{D1ec z-e|||*LwZM>HlfIGvT-V6FjHwhS#wY=f8A_gaKcV1#)yF<1t)g884M0fwnTEaLiiOHLheN(cwUIVL;jLTt0w#qYe=xB;fh|5{LL+RSB_(ow@ zIjrxHmm^V&`|KbhVdeW8V&{N0^~CC=W-HWH*H@tjhE?!357&ITaM(yRvCuWQXkZUo zj=RdG>~L?%O~U{T9zh`^xw*a-XUM}Qyz5O_7o-bqyMIueV&YHKM;vv(xqBz4KY?uM z!43%}Iq?n@iQg^8CCPj3bIzuk9+1|)o*`4W3KnMQA0qyj-_QZ476~+Q1#tPdEBfXe zMAI%x4~)`ZaAchlu?d4y&uD%lGjcWK%(oQnNKFmyAxsPgOOIV5GtZO7ICEgKNM3{B z<)?HPHvZW=@9OQ7MJ-%GvA$yJ_5Fv)mJkx>Z*J;CT`41+QZW}dROtQ=qUxXGURI(;?NE~bu>z7sAz`e%(7@4EnPo=;JrK#jnbH$(3cXbC0AAVbg(PaDEOElNYfc{;(iHO&R(Pf z?9cwc8r>g1s~5PU9Wm~H8Yp@6#P;PV0jZOrbH@*$sFl~JSr}ZkrV0Z^hca5#dNgTE z>{NZNWQMI}vE}(izF{N+dO>v=pd51OuW;`AQsN+jWO}AzB))wi8n`GWL*e>o6rm#o zV{JPg7VpPXT6*$R5W1JRU2#aFl*jnj?wqSJgPkPug$)-DXE}s7l?=*L8xr)dXi=>f z?1?zPaJQ~|m!kS-tQ5r>K;kfpRMfu)VN6+s;d1QrRq(U@hVp)Y6b3i>DjFCoSm=&( zpuEjLb#tSoNxC={A?_B7GZdrzGdPplMV#q4%o6uPm4(G1;N@RbbMl221FBDu7>xMC zmh^ioo@dpor5yT9?^nx!ty=nwiWw#Zw@=TY16I>sPPqFMy^jdx{3;FEekwvB zVisf*MA5MZZPrT-hkWSLvo`w`Iwr0)w%g27&(OlR0Vjh-!7C)h>dDA||&WHG{6Z6`tccMq|^WX14P%_FxRO7rmBUtex_nX>e8 zRBEvz;GMMKh@YVTJG?&}!~(>8@e>NBw8&5ElV&WyOC2lK;EA+B1VN{YH67__Lsdy@ zDU(RN!{FWzO2s27ham|co^GlE@#Drpw!3Cv|sQyAg-fc z0UqQJ-JORjChK^iAc#P_$Y?t-ME{Q_}zcoj%siq0{=uzmKG~zG{ zM+C3{Z)MH}C-sc|&p#F{&lIVX_Avf@{XDx+^=URe>mn9W2GB_{4ZDQw=R?i(r8N9g z+L7f=AMm(Mn;X0MG%miK??va=|H51&t<6*Pt_2+L7d?dWz~R&FWBiYJPl~6mW!^8x zriqAO7p6I+zGN`ikfvHVhN=3ayMWY=u(N9?2~s(>rux$a-87Vu8czz&O!}zkn|jnhm6bU~hvWd(|+5 zG$L^ zDRhWh!rCYOiO5HDOx#0Nmb@$y^y$>lg1)OYtF{NbXWJ?_H-Xgk&yUtqgLqc28AnoPOcLBO zbK@2nS(tv_??-SM`jnQ`Eq>8uW~X11^QxTEc%^jc)kA{Vh*qRA-Ozt+<)>1%=bZbO zykhgnnZi^{tPsEbpM;temAX<+i8ro5B~4-c=S>GBu<=Ve3+eHpY-;S=m-$=B|`Z(A4Lt4pq;^(J+T%G(G-p{pvXlu z);Eyqg>|?SdFAndn2eZIN|aU^m+eJ@4521t-f#MS?=02nd*ql|t|G-u?k21>X%JAC zzvlN*=AcafPomahCv-w-FuXWbU4k>a9&_0Gh^?_`Hv$*p=Fok&?_cm7Jf_??H)6(S|NMtRsGH)1d&4dYZTIB22DvV^YctW{!Spww7SdS$fUTX$6CGn={i8Po{M z&%9D^)OgXPBtwP&1&Kg*zcwI4NMB0(Igh$0oG$VWEhoV#A$1X-wqjz4o8eEvmsRw;LpGxDMS*X|ZF2Ctl`DQ1Y=_xZ<01-x>aGtD=3pzYW;Dw3vu4Hdx zh83dXn9k1L>(kxH84~Am$~2d0=!E&1?jT7-pTZpn2V5XP)T@*$pc6eqYZ`(ayj|!b zlWm$;>T%FXLyvG&?~kR&%pr{7Xe_hmK@v-DhX?4bOnIlO1?#XkHo>c{9aJE=pjIfx z?Ys&UDy_US)SS&Rsy$1*s}jIuvzc<>t{nJYP{rLlAyOtAt)fvUHGM!5(`=)aoWyIv z%WMKoPHZlvrnp}t#0*+e2B|*Ru3U{Uvth;9=uDWUF6ny#$Lwk0GE@^0?25f|SOjWs zLuQH8K-)0emtX`hUnL0m%9gF!r5wv$cEFkS_aT7|7eg#5FUZV3fC@2Hb_Xy>EB#ZR zlal(JpG?YS)PuNvHGWcrfIO2TPJ5taHMqPX721#<>MGl22R-sqY@UipI29%9X}3)- zkv|`sIW8Z;5fO(OZqUV--*U_=-Wz6mkqq)db{l@E&4JhhFs2kQodhn4eDKnkj-wOm z<{bBe7XUg(IKTrTw#_ECss_B|+A{4+PewfgTc6;c8+Do}PISqbL-N`C7s2 zm91fhMOEagDPY>w-BH(zsqX0O#vmi1^xpYG3Tg{T&eVXqunj6yXmuZ+Gr^lE36}0k z!4s)>4y8XyZzUq%8+kJ9A_M7+DVB=AvZ~EoE~0k+x9l!|2uoX#n{oaLFJ+-VBjXBG z2jXUIrhx(aR4wM>$s{?-uNM(iBAh>l2Tks?nvq{w-)ZX4%j{OE_@UGso3VDWPeHHK zP$5uIYTaq!qkK{?+b`Q0QWhIaL16h%7L)Q+4P;u85V<**bFRi2hH}&Er!aCWWsbp9 znr|r9+YiJQ%gdYumnG>EvU@aVWW<50Ljx8^81`D=bQwdNLgmA#mh8Oz3Y#e-FK}6l zK_y$jY`F(J#v@;!5@$C0nMPtVNPDD=vh8fpi|-5U4_b*3CB2;_iuiOYMp!;)=t@*6 z#3o@7Eg*N1S-z(!Lg0$yl8p6s!U%_4&+%3vN>Eg{opGu|KVe2ex6E#f02$eNmjrs} zD+%2+PDX9B!<(>qptlS(^$|fN0A)$w6uz4PAmCN|I0T2U?3Tzy`2Vx_?%i$M+T!5+ zJNy(lQhHid^_gU?t}fX83fbWo$PfGAzJyn>FT*JCX&S%`Qa@vi#0xy{#NoHzB(+~~!bxjy}NSce}p5qkV zLx4psFG!|q$TRCYiS|S^f({h~H4m9B#1#IIFS&ogQVE>XZQ`vL`s1T76JsQJ?XJ{yy7w=AI9;dvl_U_SdkYx^;J#lu1iHLe zSp<4~%QGehPM5fHylEcCIAHK(*PC(!oy!ZQ9u*>=DpA~;PDu)8eQ#^i+iYo3d$780 zkj~vSkCiQor{FSVw^ev!`x7o$TsrQ|!WobcAEB>eY6qkvIgG7-$LZ=+UNA%z8Cg47 z4oH7PeFk-SaLAjjk_y!`2Ac{la0bx5bmDQlum_rCcxdyjLo7bW+)si zC%DKZv7lZ&X7qEFV-!m=ZyK*5Y?ssHY)dU{@P2hm1^lfKI02sOD>r1gy}is#~!3 zt*pW|7y(b2$ixP&`pZx-b>%C}NSsFzWJ|$)q&TAPQs0LWPcLa20Jl&mtV;^;qhnd! zc&lQwsyf(3P+hDcEPE{4q|Uh9I&dNw2itPRgz91 z2jHfx`r&cK(7K{e>ToVs)pE~NXVv2!!D*P@l0%Xm)l*J0&YX2@5Yptqqy>g9Ibk#q zYA~>jHA4?h5@B4JUrZGc<3@SxXNrllJpGg>L8GT%5-#)hQ%t$)sp7{ua9UU^{kp;L zb3kt7h`Ag)%10}Viz`=O6F=w#x&%Wv8cxa8ZRr%Cwn%#s7D*1Qe!c{m-3WqV{i$zst^|~8|?YIW*Dn}P}yvTm!(gY;K zZbttb?TZxF<1m-=w<^b;=vgA6QWr*q=@Ryf?o+vLncszSuXgt zBKu9{LTz@$=V>uv$^u<-E|bJa34>X#V>!T4K#3pmJkWbWQty;q!VTV(&7kpP9=Pwh z`=dNRlulSNYQsDOuWVJ~mP>Cq@2vjGA7CEz46L zIs@iNo_`pUNs=HdBut?gmONc5Uq*9H?$U5P$r`$oqbSU5jS6wKKcM)4Y&s6eZ%f2> z9a0I!_ga`Qlw8a*F%(;;5klGh+s$t`9YR(jUaD9F`9iNFdGh9Q5zGaJP}{<|A_cn! z$tsc}HBv17xrtz5Hl&Xajv&A1=bHB4A_(-^ghdHUg_kAEOuNqzJoAg36b zRt~23#L;h!hEbAi8U%&X6zrk$5CSI@CL={X(EJZnUHO8IM$FF!r1O@aDgfkbt(6#% z&f!0E8gXo%N zh0hOwr4u_WEYv*fCj5{+qy;(5bQ6z^*}9aU!b;&_axV>@n`PQ$lWfP8X~MHC?~mH? zW+M$?Jd=G-Svm$2;bj_T%m%z@Upa0p+99nCsvAjM){ z1tM+8=u{?at;huOUZ=D&u>e>I>WhXjAUv(8B}@m%Q(DnVikl4`2)$TSXXt|UZr0bE z4ue`nEMpP=PJ3BBe|qMvddhh9jqtt^o|yTAZ-P!^VyaS-(e+A^S}HIokTs2tlowNl z@}Qltb%og(%Bn}7;g7!&u0ee<(e(vZ1c0zo(wWceD=&Kls+=_#6wq&1gj47Hbd|#0 zMeXl>LLXVX_TpRN(j>V?p18JxC4%6Q`auKb7Vh2^CR=5Py$@`a)4{HStXgi~8{?{y zuCw|0XiUyoXA)f?7bS{(ALwhz=IKHT!zLUqNjQ2#T&@J1l^WC>9!OA6Uc(vEMkC?8 z@d}(g$^}|*ZJwuDwd)ocA~VbA+g`D19xhs_TXx!8X(($0->zG3ZE;~OYm|*7vkkk; z2(3pfq@$wrguxmEugt-7^n21J3g;(l7;TH7HT`@g=_v9Sk`?esr&tcf%Bkn@RGe{B zSC8(xuSZH^I|>wL44=L--fXbS0;XxZ4q(0C4h2hahjdh&+RjR~P_3u8_8I`zIXk04 zkTN08&N}NWHIo$yd^@D0zMqA`YCu?#bvvsgqmJ3-y~Js97)9R6wsWhF8l02vbTs8KaLdMnr916>$*nPBu@jIW6^=YS-w_ za$_<}7fEs}9y=Foej5lY0o6!HJ4fABqq@E2G^mUYJ^s6xf@bpH+~1#i4pvKnFaK1z z^y{qjS4gsZWBXslFGaWzeyQP5+Aor^FI2Er7cj1Der+LkM1XURmWw2x@PjO1rQg870gWiNjd# zZn7imL%^lMVv}bjrL9Y768@0#JP`x(;V~vpdi=5M?jhd?+SN5tIhvr*jHN>{0B70{ zk8{CFHNo_O_kFIR^{ny&G!B+%cy6FBfsw~N)?O|@oxGOg?C3bF9cyqOI%Ae~KDHST znj*PrJsQ%^)oU~nVb^tUu^hh-;~{+k11*32Xsqd+4!z^;R*riq< zHdtN_V5z${CJ3>jH&oG4GuSjupP8fzIJcUh>jHJ#DV2k2*$%61z7~u~SwRJYXuLd8 zHs)L#XA88PUj>bXR3e&?#l!ZlxiBhJoP`k^ z+wPZ^H>`2Bl<$s?56Xp(k6sop(mdW5KM9s9N5wp11M>JpzTJ=KkE>A-YTs{p-02Gq zmDyE&E~#9(lc7jm?PkJnt5v_!2Va5ny~=VVke53Gr6G+J|6$|hO)fIP&0~BEKDzo! zk4vUuJRqCqD$kUD8j!u+-R)h6kdeSM=8l?684cdW(fovSxXOYqU&Xv*3}@AtOCIA5 zr#qWZH5V><%v99q$NGg1Yb{^EO-~m5s9x!7)-;mn$$(3cc{eE+pEdmkJyt=$Suxebb@tVUrr4s!402 z*hIKtbDT#J%3#_ugH``J=VtlXw5`KhAzSdA(k0zhSjkk{{UJ%SP2_mj*#9+kJQPeti+hvJP;h zt~?T+*I7^9za506l%BY_<~wAXxox=g3mHz+tkcwB1*;9CwnLp%!OsPABmHsFSQqL+ z8t}QWe6x|;hD(=l#YLF7GbnRo@Rq4*?2bX@;!26)`3)jn+!(})JYk@T@YC}i%lw`Y z5%Bo9`mH~`1cIrS>s_$T4)(}`G1*X!YPwZ#d67pj zI*Hm1YMaFYk2Cfuv+)AznzuZbVtwf_HXlKkL;9PWn^ipBkxUQ60?GT0+$f?x^P29; zif$057Ko6Z3g@(8HQr#m75Ve6vi6})JFL0tJbDen#f=l70Zd`Vq zS}PJs1M=v%gTs^eXRnT4AD+G4e{=Y`UJx$*J^Jl$hyPeu4Cy7bxL<42DqVa#efQJJ z!Qt~qYojm@)Y1l+d&C7Vnh}8j*rZd3l3WoXsf;Vdh6$qqabwb7xB4;gUZlGVnEyesQ zH5ys0^1f?iJF3d_O2$Mv%+ZCYGBarz@n%@xihEdC;kulA$*QAp>cC(XG?_aR04c zVqa!MEZ>v=<2(O0+~WUQPOo=97hl}vwFdEBc)T{C;dkKh+5pvEdkZd^&fk;s z%C3|4%bWO48mB!an=JRCyc4&=LRD}lZiOw)-Nvo3#t{FD@Wyw<|1Hc2xQzdM`m~z= zZ*S*;|GS^(tMGr?{jcWHcXuTF4m|ai&}`D;sE1*m3`xJX*IawIci=SF-gwz(jyOwH zTnlWy&6eIfc7dy>kZ*P=YbbZ$?r*i+^ot<9hU?okJl7hz&_Al=&-Wr^3C zUFiBgYl(^Ubvuu>#x-?)|AK106}$_U2_JtbqHP#$o@%*Ht**pcR~Wjz-Nnv|wOR78 zQxX0eis|zrKLQVldQg3I{7{?p3+ zkL}Iv&CLh??>?T}-Tx?up{rj5sfDR)WCx42=lMKk96Wq`RXU>sO0Ga|mV{&c0%WHp zVq`1*DROs`-+o)(o)zS=fe-zK4}ql|6q!`)Z@+w9dF11!SWN zvITK>er-gnjvkpDrGw|B(;=03WpeQx3DODU8I3|K9;m|YI=OBQPj{$7y)>+JQ)Iz4 zh)O&qbH=RcxTgnjh1ulRZtdgwH;~+>o_9?3@LaOx9kQ13|3~n+eKhM9_Uy2IW-QNT z8znxX1tUv>-`ouw@w>RD)yhcd1iY*85i7XrN1@1ADO2PM{`o7j88!fQK0{|oSPBLII$g{7X$E=6Rq*RB`9CXgm5Rti)M`)1XlweV zLL{GUkVouOC@^HRYO@5<8vCay0z)f7Qk)hhUFN~bOH7dn$s6ObM~)#TI#Iii92ZGx z-|3{ZP-)0UJT>LYS=Kz5x}{_LN}=fC%`%o^W;>i&R4NSfFoLYC3H2}Nm4NN6EB;}zj+#tohSe0bjjao8uDC_qnC%m@sgDP!u-tf!hq2p z7Ek#vjyDrN4_NQ3Z|)VIYwrJ(!~K_U4!vn`ojw-t|E<0L_Eu&8_n+=Ood5UobZg1I z9cN$X1l8vo6`0Lmzy6?;A%9$hD++^NZyaWmdnNwY}}4<6Yk9~#8E#$llJ+)GSx zS0@2ouh>t>!3hZkp^@NP?^z@#HyTJgFo6sql?>kmM@g7bod{gk9q}mQmzW}*#gg2Lf#Z^Y@SMb~FhcGCw#sZHIE1X+o$M+4^F5U}-|z0F4yd zB`EAu!eJW~K=iksdYj&+*RMfwkq;T#qYW%ede!*HWGA`Mmv;q#!UB0zGCzX@mYnN5 zP3J41wmA_$j7=#jNJ&f}vyc+1y;Fz^S)zuRuoN^HDwFP%USLK`CP3SLu7gFNFBz`f zF+9(#3s-n^R*<#OW1dRuPR{GWU(VMZ=e@OXtQsm+%~?+a*608S%M||5@ML+Wo8JoMtr5^wMV0anaWS3MBDg^gKSebZM#B(W#7QQ|W zGZV&SL}wD6@{E-PAz}mvh|Lccjn{LkP>;x*=LUOoo~K5rZd#>UDh6=me1qVOwBcMz zk-DjKvc_gCmKc>COlh3cD4G*LVl$Q60XgrA0R;hbEN!%uXYM2uWH zSHzVMQU7<#zo5Xm;W*C86@g>hySZ!1LSU5K&}+~1e}~~!!mN~RxnL=T(E)2 zv!m~+C~>@!CS6iA#w#WSS($T{lKPp7CH7+Q%LAdmEh5g+!ZRFaF{OYY}z zu}DJPqeU&T7Efd$y!VDd{MHTb8oCPZKy6_^6OUi-l!vBy|r_jm+LZ<+W=ED3s6?ME7#S zQfBQ$oQotMMxmHgbt$e~li?`z^(yfsYlYnc(a)nSl!r;77E)Lga^J-~&@7JnZm9-< zVU`3W3VoPuRB2z4^Y6CzzN?Ny#X^*0F+Q;+9!Hlj%nl!!6T-Mz+EH1|IrjOpx z)Z!As*(xH1%k`Qv)G8>_E0mG}_vN`d7lGY8H@Xhlffk}lP?Ulc%2J0hvM{wEMv_+B zdJaXmXQ8BdI)s$aWETjf8U>v73;VBTDwlw~;=lvnD;$f$$V!-wMq1F6+is(_qm**8 zu7`o;QBlLFle5siSaG68%?cnzPOTnuf&04JA;hXJ@S?RoT4%%p-M!KS2~7#h8imE^ zx=y~>RnV7<`arC#j~H->1%#HFYZjo@>n6}OB$+)|@|4oEi94FNP{<`b;ejIJDf3b3 zHP=g6^LYAf2RUF$=aRLs^b3@X@+=1-AH|*9&ruT=fCE6WlIp-pFai2#N_cQr>lz5; z4T3ROY3fZ9|K7#I{Bx{|#stJ)4?VXYK{iiV0e&;sz!^gLQa-9^)I(Ids2YY&f>J`A zMRM8V!n(#vT$n0%C3r8`oaivPu$BuA`uuuFnjFlHMt$Aww>*_^d7;^wG7AOZVHBeB z?-BSh(&QY#S@=5beiwdivVj?k4_(`4>x;xe7yv;%8w&QXGB3?)Ws81OdEW z68AXTu@ zHW*D%)2|3(5pA>;lk|6i%W?os32Y^#Cg#^jsDPfdKx*!_YgHqxY}hd#IYqHFV52aW zJbO{NiK|H&4bWB&n?Tz(RFp`=Xj@|YEf68gw-ExlvsG_iBgQJp4kWUEo-i=DSY%0S zK~~lQw2-N$9Kb-W%&RIU74#K7nWD>N2!8u|{uBuTdZtL6(LkwYXmdx}wV}Aw)nsOe zbL3kzo}wq5_L2>a+NKi7U(U@nl8?}o;bn#^a{l3C1$p(nVYU{@0v~9%rH5J@qYc1X zQnGX&2P}<1OJggJj(Jc~+#@c4>73G-j*)EJU97iFcthGUqf5`HLk6Z#(9cSUESM)$ zY>?2bJn&J(1IDga(|VO#SYcDuv1${T&GzBw^kRBACfyiBBvjhF2&tfvfYeHjc>rcm zJVM*Y@K_*j(mcNPP->b;V>wj%Bx6b>Q8*2i+eL|`vzAyv{+gP$W4U1K@^3X<`4P4@ zf?HDlp_a1No)vKgngen(x5K~ye7Y5T^OS-dRrd=zm7wy+(x53jtL^z!)}0pmyppv_ z`wtc3*ov*Aoq}H$u0n7?>3wI2kJd?jyZP;A36z`7B?8zymGz)JZ4+5Ege143a;~(v z!c?w>qk@fGFOXy5lvg{-jQN2DxV`K}aM2Jhp=_f$WmCzh`aBj{O2O6wx61@s!y_MN zwZZc828=1wFb+gdKqKpbY-NL;(`jUGi3^Xs;wdf$C66>gwo!I^i6Z3v;Z$NwQ*S>{ zQ@Nb#9+r;1b*xZ;CrZkb~oa`061hKh|l2RSU2`dVQ zWe*!2WYpjT78}b~o&<%Tj5f1FO)wr9lYxOgWrW3WSTeYUALPIg$fhHjfWlV|3u6(k zZz%Y&zOlF^Nfe?2gr@;ZJ+(+2=gIpp8Nf!C44Y(Yk#5!**JqNoy1HQVD`baXAV2Jb z`x0Kkz6_(lr)dB)Nd1hZ;)zEb=ajL+b=UyHNHKL;3IfVE$qY)o# zud$H#CO}>Xqyxc38jFCt%}fXOc>Lly>3dsy-ln@ro{_${-IsqpZmi11fIwV5>#JIl z^0TCeV7DA-2c_n6R`?HW+gi^qMuTgvs8rN0#_ljqP_?CB4P8Q$r1-h%^*6muCn%h6 z24o+d^pGw}vMFUdONXx8op9N41CcIF4z38tP217AVDn3!2H+q+NvPwR;3KZpmE)Su zEQSE)Nl(*bXDSbC9e2+qY)ZpuAkVlgq{16BCjR{wmWhn|7Z_bz1v|k4^1Gu0poomk zSnP@kW0QH&)?~!en8pG7_1~vVaFI+{%H&8*V-w&6f7u#2DkNKd#5%H_9PlEPDH^t{ zSSx1DV&3t#yv;xJQFbl*Z!uuk>0>ed@9*|^tM|XQws-a(=>L5@UE?K)ldEY}0}b>% z2YrgwL6Jo&1Z^S`9RgFy!bQw&LXiBbUQSdf&H?Fk9LLJE{@HOdp74l|=ch@^XmC)- z5CihzqvK$pPfTU6B6S*y-J6DK%2O6JfZasR3CZXfDYx9;gd}+s>3rcwqqjUePMKhF z<~R`_zh*NQ4akn;DEfNBq-b{}C59P|2BbfA91PTi5tjCLcD5a-67lx4<0NUwA)0PP zg}{KV5)Iw1aVkPz+^X|1Sp%b&6kXinWucJ)C9uz@Ly#qL|E@bVM^n?Ar4hBxe-MU) zKsz=XvET7?k@4xi#VKH3p)N^zB#&J9t}|L!d%zG;OBy#3;jrC;8Z{mRPM7?YM$i`C z|FKTH;?Dsz1zu9%k*-2l1P&p28sVWk=vj2mQ*x=4NcYU|{9V`&Y_$2YCRgDAE zmw?D~z3*-H;ii(%A?>x2h2h~J^O&TZ+x}C+>5`O*r6TPRKdr!Y}k|Jt> z6`NTH)Q7;!0A8h#xwfbEG6;o77P!@CR9t{_J-WEK2@hOT-}u_YEA}aK8bU^cise}6 zHaHYMjVQ87noRgs;0L57LCTW>`Ou+J)cLp|xW5WL-58I$5{bGG0nU7%Pm|-6OJN+l z%H|0hkdr*l!YNaP%5l;m^>GQED(^Xnwg=XWlkfKrtlJUAZ31QYl_P?`6ex}&kxgxt zFIp^W3X$cM<36`FTlf503Z-mFJE9jFovJsN6z@GKdxT*-w3$e`oXM>QmS>6$g@T}o zAY*~$C^yQZ2)(1#zSMjmj;Cyjv8WVw{gYtCr-BX0{?Di6@NcK&X#Wk6BRs(>U<;&H z_I8s?+oPAlqf`1Tk73`j3(Yioq3aN8p#D-p^Z+L z8~|M-FW;UPS1-V=LFM1q4%&6{25NaHhy~v7FhmjPIZJ0MHg*@HaQ3||Z`<2}-vaU# zGwW+C`nL0^8*s_!drF04icRNU5DO3MdmtTA{e(fGd=eB7tIOt5Ye) zf>>-yf>{KlR>y`?7VCEXB!?Y3Rllcf3a5L2>)SUWyhBa%YUQ2cW(*@yIAgIoB8JTA zE+TX(8?ltdKC?vxnuNVeo?b+p2Cf+j+$@^)d);GbR`&wGB$Tq3ADD`lk7q}3!f2FD zezhV)0xZZRfwSoV-ANrxz8%u&C5!0%l=(ak#DHwegGkaSo@eIeZh5?tOorBg&F`-R zn#!O>v`SagL!y`lVC`ftUE<138#3`9Ig@kw__wn&xs1-vp8r?pKLww%Gh{H%VEaAl ze15Dd+9A))D=SR`dG^dZQ6*lyFeNmx(dCsw7Y_O2tvu&D)A<>Kt3W6UD4s{bEb{5v zvuEBZjN`?N-=015fU$W4O{e0;i_hz1I480Y9Ms>>U%Y_K(_MmjHPQG#3de6~qKyA3 za|`;3_CHDCBo!|EqC+zZ&l?KA8-#tzq~IMXD~+<;usD-2Is_e!B%|fP4=ntLnHCN3 z7n~S93Y+*C+GqG6)zOvtLS(1)Hp!Y*f)F{YYo?JiKgrK>LC5S&;3hiDct)e&Oo_id z?{qdAp#QKv{kU$mT>LGikQU%pgHAR8fR#a-@__y2c_$1a*4fZM_$3W9^HXGuM%M(l zv}W{YW8)f>z>|=b-xT!+@U>jrC<)u^=pv;O2@l$dsaVosG{V zZq6j-wU!a~b0&|iFdm-?mYogf853V<9bFGhSWgPVH*ES|t|kTE3IGw}uHN?kjD`^% zMr`%=|3y~fWf%wiazM7HMxGcn@Ufa3*xA{3w49_O{~aAeQ@;K+Fs!JPVYGSF1=%v* zfLytXLLcYDDD=s}K05Sa;FfS4GjzMzKR(iN!3nuwbGv35&ZL}i!Hn6tR9e{#d^lsO z{WVttIXNRNo`oro!NF%n(-5y8frEs)2(Y0}=P6AD;i+8t`o32-VN*f{OdKjo9Tv~j zLE1Gu@%uN&XD^RV)HLYa?Dq#i`V`^M>%5EH|;&bKydOYsQ z0_Gbo>z8nKji>W>=>Z6`rA~y10BW(U2c3N!3io{K4)ZvO7>upl;j1uWO~XYm>C&K8 z&UHOmN5SbIy9hEz%2?v#>fefTKHi7LMG&16Zk5*}FE zKR`&Kl58qnRYWeTQs~>9;?G(Gc8bl4Iwc%6N$7WQHwUApH!1R&?Al9+ShroqFVyF^nM=#6sm;t!e+AF8O)F z%7`h3y#YN)DU}YRc@blQsY}us*GJ8(FcsO)VK(^@8*;iuuY=Y?PSFt!n=N?Y#&8pZj@A*>YO!zFJk^As!*0KNnHYw3Dpo9Kp3ub}6Y!o-quYQ_wSg zHi1X1_CvO80VZmZN|d-J)(4dYOQ88OzQ~xZ<{}?5KZ_v1mg}bUlJsa23VpM)2w0AZ z!b@ot+v=j;%xi6;4wKQaEo@hpYH#_g#00oqYIzy0Cj7;u0 zzx~#G;^?>s#l(;zVZA0QdK{*FHc8l&q9>8-V z=MsbX&?JF@c+SrZTZi(Q>_ZMp9?w37SO|Pt3Zw$Vhr1Ja^8xItdfA48tf}ioFW;WZ zvT*q!Vg9=39F0nTNkyY?WRPBtimhYrWNn6R&S})O*mlcoUw*6H#wPClV+x71%RPBT{?1dhR#-v|VGowr|?owsWdn%v0WRu1e9`m|k_c{Jq}R zd~D-D9~G63I_i&FV&ld}T6wM?SVpW&S$f5;ioPTmdc|#;66z$yT{#I@XPvB7_*1-j z)L+b@s{gNB7*fR8ZoGc(wiN~Oees$4wS~L|&kf~&pQRa6?#7J&4v%YY-Prw=<|NCZ`$m2YNG=KO6yAmOgw1)#m5VQV1ToI50QFcB{F? zS2@4RiN+fbZQEiAr79OOWu?Zm+^$$jv`t7!!x@>p{|bgBklVS=n^K)i&^zX%Z?(Impa(IX2a3v!6#YkXz!v`_kHwE0dHyg`Ms(CkBKKIlUT=TTVw6D#If0gH^@}Ig; zmhyfA{iKNHxUJv{Y60F|;Tn zygrbMtnlCCq9~n9fAn+VI^@{qM!}?A*&dGEI*QEGBvkjdTXF0VQbVzUN8N?4bRcp_ zd;`%^ux|NQ?X`x}r1bO(oZOIGXwY*1&h=$~w&yPCzjn@YQ$_P+ zL;RO}c^>Hhm#6=geQ}qR|CW$E91;KO&t1^}HoC8$W&Yn!pY|*Ce{*+t`+@%7$MePL z{{!J~Ap95&!!=}I>0=hddlLcGc`sK=za)`T@s47JS*YF zd(w=RKg^sy%%we-f5SDHw?x!e z#7l|R+%bCo^UKf!wf*l0WeX40TVUmjoqX-ssXEct_?Cjhsr$Q^N|d>)MHgO{-03X> zh|gync zM`W6d3dup1^#Wj% zDQfLn{K@)*>%v`GuqFwG#2zV+qOawXXnBG zcQ4QHits*A+y+9|<}}d8He2adK6IWy?j-aKP(RD0=!W~>$_BsO0bud|?{D?Ds`>wR zw;t^O_w$rBDv2oeWf45LX3&bqdyUt&ilf-bel11gl1<#- zyzbdkPPJ$kttVl;i_UE?0+T;B2@pWm-m7=cmir*)3tIF(`KJe6P&D-JJehnekGQhSo~pnbi1;h_Wu! zyk^?-=`U)twN`T{4JqwQr)1@8D~DRoXI>O--O&x;H?-3TcMajCj8!emp%kip*^02E z#y2kryTr2fP-(niVNWqLT~X3-zUoUET`7&nY?WrpZ2dw4S$m=KwhfwLx3jqsyOjkb zvVl78m8zEdcbmzc$*fy-VlL}B%GY16ZWO1LtkizuS==tF2`QYez zb4SmXykGyCuAXhk_(MB;{_cEwuJhCm^2M&4H+q)&e{NOn|J&P}JAd^2Z#QGC?q*Ea zc$WD8wD1KF{*_>gas8;pz1jMUv`FIZt}r(wK5qhVA?_x78zDQ~6fdO(esY;1yl`YEOr7)tM!L($Mg z8IF5#JN;vtKyFI^#Y&O?@L59tx3~BDHTu8v5dZmJ9*h68m2Bml>~glNDsuS_nZ)(g zb2fr%uXkM7=+ObQ1KK**bxQ|KxmIy*l2w(h9O=fmS%FnFnJ-Y$31?J(Y?t^7S8{;O#Q zW$zESum2>^4f#J4+4?U0KX&$ZYW^Qjw;uSv`*<4tKlJQAxPLsje>}K<+{*pq!Ruo{ z9-KXXzt3Hr|4nWqniE*2|KG0pe{5~`E>8W{(CRa*P#Cog#Yt$w~q$@4wC~c<^i+*>WPeIH_QNb z)BOOcKvutB%lqAu`|tl)#s7D=8vpC*ga5}pJyreB?ehOUaCfCmZf%Tq-`5-RdL<^0 zeacuYXG!Z}Sqh!AI17EK{*dwY;c0qqNdM8yxuOfmGWx%{S>gY-cD5hlf8Wp3uJ&uq z`-q_e{$=U=9s3(*^F$p8Y0NSP=V=xRlMGUY+iKOjvNQ`vlE^DXt%|SJR{ny`9|-Hi zW1bt+|JH0z^=BFV-`%R_|J~W$e7OIAFHZ}Fui8(W!jtGVKpP_eu6_bo$3|0wissis z%r+FQlJwuIp?a5}PinQuvSQ{(VdOPf_w)atth^ z|NW;`{m7#6DRTnxIrQ&EQ-iD;zPVt7slfalCp?q;fxW8 zi(R~@ao}`G%*H@6$Xb%JQTR#cnET7RN8ZKJobVWGN{~pxQWAwR^BnKx>Dg(uw2T2`6H6FYwiD& z!~K_U4!vn`y*?K2|NhR?ekK0jR)6Oq|MUGkU2<&Zne3}P1kO<`G8#pKWD`a{)JuF^ z0|9Q(>y5)~k`FzfPkV(9(-q;kX#uU15l^Ab1570S2Cd*^|M zwj(~Ds#|9Y`P#t-t$K}!m#4DcwOPtoZ%U;3W<@#VL`)3}CFk`}Ttf}aG{hhx;nIBk2eq_;!@2o*UZ76C z^TY%f1t3rUb0$mmaEZz;X&jJ%`;hPeEwE=WR<$>9ypZmsKE_a$`EYXh`f&gB@a*mW zo5PQdBW~Cx_5kqSV2*bmKCX33zt_o-Mf_3%hNbP!myTDJTVcpQ#o-Er&g14`<8WY& zr)NL*KSXiu?SaXH9XZaKhR@}83&Z_C?soD-dY0uZP( zVaVk;WJqXZeo01o6wQfQV8e)!Y?AVPJRviX`^b4Yk;OSQcV5b@aW1JDBT33;z%Z$r z6FOsL$XHA+SdtMcbX095G78vD3GA_|8qGIMCegKB#cBxE*MMX zBFQWyL~{U9p~41YLl=5P00OvoN~!EpOXm{IJ^2Ll>|mYnsHFa zlJk>1&cZ2si6cKRW_=rH{bkN%VJbIzJZ5r;))b<0Xaiu&U4)nPkkoG zLU<~V+l-Txxk{B)UK@DA9oTd1IIt;^83}*Ya|!#Me9uNwK2k?O7>`@_YZzxK@8u$l z$7is2yMEE0n@t#=UG5Ot(?lVR$LpTNiC_#!WNBsc1XV3S5(AdeP$F7V07*1A$V{Hl z1+9E6$Ol|;pq>4==Jl+f>l=ipB+X%(1*diQEwP$Pb4U*3 z0Mdgy&U+5a(+psPuvp|NzBwGa^KBo8(KN^3@#0>C3jBl~+O z2>X<9;Pzs0)sr`h3dNnxL@b_{bzI0N+x=b3-$K< zOeMk4yedzDkW6VJ3r{ryqd6Qq!1>0E1;D$q6jt$uq@_S#aZLt@h=eCOC~eQ}Ou^?G zTgNS3a=4IE&_ByP?3*8%BJn*i`|IK&@o5Yj4$fD`NDz)jER~d9%QsR4yi@6|tF%<| zI4IpxUmxHMmqk#=Ih02ilKW67YD+FdNrtJMvdX+=)USn2feOdY3 zBZumM9L~))1#R1TDZ4hD_F0;<4cM;|eM1dF_$yXd8w&_aX=@?XhRkouyPT-1Iz)+- z&i*53>3l#Q|8{m3(zP@-- zTxc~BS2BOX!~i|oHG{~Qr+Q4g*FA7$fs-1o_VJL?~50|J$vSX)%OiFnTi)L zKCeqm$U@*Z`~CdIi}m8TIn)yP%5z+k*;{70B_^Xa&j&bVteEcleCbviXH0+mea)GM z>I=@AhHLd%Gs=r4v(`FUC)Ea^aT-PZk|aDJ2gg6bY5j&xc{;CjqjgmGrj??CTrvqC z%E0HcO+SlB#*=f+x~;YVHR2ir9VEG49fqw@6Mj`B7s{vgnI&s>b~c)^HPgVE zpX6t`pksC>n9t)toMk+t(Ql^2U!He58x1Sy!}j#!x@mXa%IVe!)GAQP$^cLqlqnC` zU!HfuAYz>j{exf9Ff%`;%oJS%%!b5_e8$o=49dA0B}DdbYfj)>C5siBx7*6=0@nA0 z2W`a@9#m%=#x^K^tF140#9gcmIvbx!ESyO((OM$d!`UTIFQhajSavp?XH4Ar8o3U5 zuvQdgZ`ky`Tqz2yrGOCPuG+2%uDNR4f5H`Z8O8y>9FXlP9uZHTynXlnaPZ^_Ib|6T zGm7#k@bgu|rYX7LO6pf?!wS9MKR(g|;f7L*Yr!8>Afr6?QThtAxg@-Iua@`7z91s^ zCrXTx;#b0j2tg}lgFhuho=vo(3*b|}Xx{ZLkSp2$Ao-ybZc_{k2eGb_)AU zZS*{i2Bcx5H0qIT3s1@SY|EiZVt)3$r{1O$FyW^mFopxNj~aT^k4U0Wv`bghv6HghFBoZC>)!0izly zGEyEzL+W2R#gtp-cVwYMp{;iPO($iuP!9Y@Z1Z(E4YPs#;9zuc&?`cKaOCk5VG=8& z*1<}_j(6Qvzb9sn^dToR8f$%DF^Ek_zVJf zf)%jK0wvv+9Nb-tIVu3?w|?1zFFd_g^Gywxn`~T*gmf*$B@|Ti3g;O&vZL9ZU;>i> z>jteUHTxEp=YcVdK+J=5NPW3sKg~Vz^Q15sZxFQ4O1e@h2u7roXUs?IW@(QDXHwYk zfIgvVn28Ks8Vcf4Xa`+8 z=D|9t_+DUhCu~#YhEaf~tqf=_Q7s zZb4J1Nm&5qd$gG8SE&23bl^&3QgmT?sn}N-J+s+=R88qUUk+w(_VLe*@%V1cACe8$2N6RegaV_&urDY0cQi1PgIaMVI0a;2q>NNoCKU)(zrAlt6Cl_9~1y@ z#O2-y#v+vDCB- z=ej)jC{YPSuq3Rmfzu^nJWiPq=u6Dw(%y2qB<6%+frO>gPzn!DY&XuelqXI=*_6kEWsW|8yInkt;SW+fiJbz`5*j5Px8k&*{VK1NrVZA$L*#V{ z`GpUaqFcWy&e?i&X`kr5~_*JMShAa|D%k^;s z#~~coKc&%t{B-gfxDTQjK{>%| zw;Ry!3W3XlaX-tbKf$Tdw_ZU@9|~Wx1#;;Ds)R>Qw4ii3#2O{*b`B%h+Vcx{V6Y4{RK@-m~5Vf~PLybq_0=OEY#YX&rxQS)!5 z@gw`ipZZX$pi|q^*R-RnL`|nqF?l{u85gizXZ;RZt!yQ)HD#y3JA&{j&ro@sNV?%7 z^mq_RA`=@q7{3IP%TH=_f%-m6G8UBZEs8z{WYcu57JI2{TJ#94ORZNm?JM+}70uhd z9xoi6rVdT5D`pi(qps7V>D&5#ln$`>J@2>$FVQGHx}r0gX!{ zkcFsD2=O9S9%ZHhC2v^1)mjfx!4+JPDBL1(Z6f@yB+#+QBzM1|kOeDMN*F_C6JV@o zx&AAS*)$waG8C*wBkc(NAFs(jbA%etIy8Ax)w>5As|K=__34tgn#YlIjQy4D)xg#0 zb42SNg=Wq4bqvEPc@2ycc4TpY`a%Hn#-0||C2fY`g9^Lo)12f)-4zLA+f=uZf9nzr zpbFY)?IIzWVO8q5%o>p_rLh>XRA?%r#9K?_ddvgDV}%7g3%ph~3wyB+Vf$ol-KI_F zDz;=8A&Do0ACQCP-|(7aq)@M&8&cEqWq6_GOb~W2w>Qqa_N@lpcN#J!QvSk+cp}Li zSzdr@yCjKMR=ddJUMshL3hZuAM>x{`)>%=_7k4Cy)~G+G*#vKv!og&&yXx7el56g0 z_Yl+R_nM_v>UqG=<q4Qa;>HWvtxR}V#{fiEdTwi2dOI$|X#amU+l ztns{)MYoKw@zcrcTkv9EW4Jf~=%33$ZdtM<0sDfKblDhS!@c-vj(%Z-t^Rkr9(f7Z z8-*eo!f+ABLeem^B?%b&I#Mzqk<2cf)CDh92-ZrJ0sWukF;M9jvcSUg|NQs=Rz@jJ z$h3SZ87wK9d(Bd3mnex-Y4;SvFkz-07)Mk08`Vv`3IG);*Ffgr73&7Ztu1p1;z8of4uXu)bU9cz-7 z@+p<)eiY3me=Cf_->8O^R5@aoL`oL{O#_=j9fd4XH~a`s!*LiFd+irKgjK3+z_Q^` zkRl2HETpiY&(5-Ga#qmNvtRfS?yeLv#_Ift<#GmoBUyznscD_JDA=iHm7C^-zNGr9qPJ4y^$TgOD(WhAc-@h*N^h7Op6X>?RXb=m)>PdEd<}}K zYe2fPn(8h8IEbkpbyDPSxD{1&ys=($xB84Xt0q%xW81yXpoQ+1k`LQ)4vU-E20omW;A? zg>6~EDZ2m^Hu+TYOKCMfhG)KT4N8)uwHk+J-$NSXu7Q6aRr1zHJvYH`l!#1uG0{biUjlxS^7he{hL?jhba z-^(!Ijfz}hZUj?oCltiUqbk4vtPABQXxuBE;%=vHPR8xj;tdAegM_OLCCIME1Xb~6 zksXYL%3rg$ySu$3H+dfKH@sFetoX(9>W8(|CclIZWk1QYIuFny{2PMubDYqYE2E zl=Z@D|4g{Zh_bai@`p0ut6|7ZIiV`#29!-wE=dSEdTHJ70tcI;mqn#FxP+B+-eUnb z5Wpc=IewD0-nx`Id$MW|zE+)apjDVp@kZ~ys`~|at^z|d;g=Ae2NtQ2jj>Om04=b3 zrCA+GF+hP{=??U%DMo*wW7tHsupJNuBVZd^01Rg=ntKJc(Gh)81nw=5*#`NMi|jo& z9%D#)zYkG50C50)mE^GkEVkUq9Q`0b!i`O1Mv^TU_yJTO8a2ykgX{x~mVl)aNKA!L=EXd(DJY02jx?M+qM8_7T zB;VIB;SJH@l7g3c8ph*O)65aXL{+a3KQTWCP*g2!>r|;T-?MZoicgY`La2aOASv1( z@FV1*(Ix>1s$D5Uh}NwQ$WdJTh-j_0i*2Jg;>u}pp=GB_?1fI4R> zwY+?mLuuKht~1EO>B;<6Kcc!OOjoKRa0NlhE{7%(!3!m{I-AN4Ldfqnn#dq}W;G6x zquN)-731ZlqO!bv+n^7(huU`O1CbWQwpP50XBX&G)oOA{-`n!Gy&Xp)fTj@cLVbxv z-xD)mCwOoJKBZyoDKBPKXgc?TSa?|16ZvQqesa3n^Vc*5-hOewGocnSEa?I1Y;{US zSP+ZN^4_;M9t5A)))+#`WIjy8Krcc`1MJdrDVG-`7Xv9GmBb3Z)+nbQfKaV0ENXqDN9jo#WvcY5M;1pvn zXhuBa5lf4G=@JHJ>j8Po73D&$Q$alrL!~ENH1Wsp7qg<_;_(gD5YJmDGx7XC*HSvY zia1ZMM#0tEZj@z#Tbc0(d%q-b?e? z(rQ*mp*UINNCF(WBJ*zD>E4V`G9PwGIXzlUY@SVZdW;uHm4F+A1@KJ&slVKWy_9Xp z73ks{H=8F+(LYoLbvk72rIyTJYIM9y4?w0W-dxEgVqVK1SEt;+VA<8dgr+ZHZmzzM zxPO6vV;{&?;c(2~YCKwSM3LeOcal}hX2v8ZcU58JuF5=Iec6q|G2AG_;|dyOfhVV? z_*l9A_YA+hp-C|%En^Uk_XnTh>&3Nwi2}>q2r z1OZ#+ivPcqDFL+&8%F1VgkYnhM<5x!C~jt|&>nIN$Wyqy+NG%M)3mn0<5?Jl)D8Uc zw73llvyAtvRDoWpz@YE#c-vTPX73(#a_h#kX<`^s3(-CKUZMzh9(p$*d={!KsI47Z zh2IbjR}&3UTiR9HQCZ%ap<0QcEQIq8sma&dnMZM$9TW>T<3tKaA&s;yJK=$$pXI_z ztI{Q}OL3EB=~Col(0}1M{(h1&G2u}#AlnWS(H~fbI&S&nN65uon4l{D^9r<=ETVJ6 z7{PVpGOyH^??eu6E>@R)dFrCQyC@1HDB?hyA(SIJ|L(`2S+*HV!L|ubKZOjYQ~=wStbNc|;S?DFlyF7iedq}#DTLp z#4E^XnyHB63tG?~v{pL-sJsB9Q8;3F)99E7jS61Uc{dutML8sF^3M~7Mt}~g_d1f8 zbhP~y`+WoMs`5jj`kB%=9D$!n;U-Omi8+90&>LOIYy%hK)paW#y)%8!9~1H&XjanL z+zR(uWLj!Lf1o?{xm*;!xbTj202QFXGAyLBBm?4-WW%T!rLYE!mY2~n_}jzE#yJW& z!{BDh$wsj03~91)QU2O7QT~;l8^r%9vVgvQ_x|wI%RXg``&bhHZF_HLubTh)>C=b! zZ};-_JxJaoiJ+yW0$;V^T)X+1Bmp`tfXW=HxDG)&Z`C#D&Vr=7NDy;;VLEbCtQ0HKT8F2%h?Z;&yQ;pUkq!oAX9hqS}*|)R@lzR?_NTZqYCmsX?q8oAR?XS>p9O? zZxGwssJEkU)ycNJ6O~H})XtwjcUm`K$)IU%xz67q1B;q~{?=1()7$j=3(&7Gc|bTc zi~M$!0Y@TJ=PbP!eWd4WQT}!-W3N+55?qRkGMAH7>?W{Rf^R>{ z*p6$m{}p!-RE{Hvz~s2(P$z&490lQ zk(5UagHQ;8wx;v*^CAMeD>-d&UXolm>2%0P9GPYYg;fs_6)e--ur4Mh)oX;r8E7m;4Mt!UZ`+^`E3# zy(f--tKBy+7I+XCRhvBSP*(`Xab(>#99aYsE2`I)HCj^aA{nS(+?Yb6wDKY2g4r!P zPkNe?Y6Qbq+*elXh+pM5(@_Wk^SwL~wCe!Wl`piRyHi-j61h5VZ;qX%e{@>MDV-VrfQRND(EMX?xqByJ~ z6)6<>2EC^3NS)-0WN9AzWN#b(45z2yErmTAN?e=rpLCWR1gv@%3$d*a#b+JlBXAO2xJtBz-r5t%Cs`b%399SY z9kc$U?$RE*tKZ);y;GGuz^>J6+o9x5AB%U3CDI?m+jzH@)5JP%ZTA5$Y$8~kRy8w_ ze{_@~K>$7T| zpJi#viiG^cCH@HpHdm>F2fh}`1uhGlwms}SjQ<-x`iB3H;Qn4o7p@6{tbM%5*5df> z+a8F9p8Ky{Z2cbDA;;hLc3^0Fu(0A+>dz|pDwMt+rq&{|kY08A+|k^*g=z9t=Ew@u z;g09R>Ov_7s1L>slh+F}D0eCY3_;#nOKWO-><)Hj91R7R1K0yyKXK{Q7-jlrJ^qi1aMb$zUn5{G$H)MeGNpnFd6IoRT zB)O74E#nTnFWc)%cMfDV_%$ug8QB__#h5n1jl!x%I&Ze|noce!_6yl+<`w&KB4okA zj8^Y4vB}Sw3Oi1&zJN3r6Z;s2hiL;#tsiANs0c5=_O6?Fbsl8}a$AFtRhfQ$7G(lv zuEeDl6~8M+W$4yrQs&z|gZlLu6poZK#At;tkq@C7NrARn*VjzzyCcvIJm$6y^}}IYg(jai~VBsqi1xiv~Eq^vTf&U z&AOgPeh!=<#Y0$MO(MBK;Be!Ok8T-gBBM|{gTAf4PP%}QI z7qr4#a8SWexyxJYk?X$pB))&H{oEG+Kg)_VE6#)4z%u^-Xz!8B{~w7x;hz8B#q%5E z|9`=0t3(+O&1FUH zlj4nfXR&s%UeOmlq3`;3yXI0Fp9dC ziZlRO@zRPxbV+W-W+E6h?X}gJRlC!*voVbyDosWxZMVIZ1Pb&}#Eb9?g}U*5*B}Sc zOn#(dXFKU#Lgqc4(GX8(L>8d&)Kqg&A;g_gH=9+u6*c69EDNLxdtP18dXC}OAnX9y zD(gN=lSSYbOK00I4ICQ#0a$d&K!n$86S_OHomfx0Y#_#yxn@`mdLWR8DV@0iK5o*j zu5hy=b6X%41v|7bZknmLB)r-ZGLCzQ@<3t>X_ZIM^sEg}IdCHi`W7udKxj0qNIp*B z4&yE;7Ge;x6a&L zx{C3Ueks*fg3zwT)@ejY=yzc&|EV!O-UvI^Ez;3kUJXqIXa~_}@EIJR; zqE*)sK}~!)Z*_W?#isgorA=?e^~JglFTOk#o#T{`*22r+1OpJ)@lUIJFzol#WBY3#4~8I*#sM2S#KdM$~QJaGON{Rq-!h zhD*9`Im&+dGFno&q37hwm;d2Yx_@xEtT1+J;2$hHdf4A|Upwggn`%&tyqZe5vx|0= z&)v(*unKwwg^`d=!)lJH{IZ7Xy7#)v>v)H+s6Gw^?~134=FRlqgB3MzJ=!v*roqsR z?zgHB4O9$C1eWV-4i>6A4pv?5>nJpAb;c54_sWi2R(5nc>945EC6|@`H)XvZ8eie! zY}w3kQM~bmu;shEUITseyGO$8nBQa*Vi_)V{gapYFKbwxl+;eME6VbYL-iU?-96X~ z`FjOpASa`;#pkW}Zehrx>=itI^}=n!StB|TtS+LnzdCffsdbjlplQmOq?)(dl9T6B z7xVL3-d=l7=66%hO9;QNy_p>&RG0G7SYEpiGPB;Ob71o{y*l%j!vqzt~u*GF8=tio-Ra8QYuru{ksnCvKs{oxC`treih#^JaDs1b+HH0BNdT zWVK6!8B4QvHX=W$1Wq@_@aqlLHE<3q8dpJ6r+RepTvqWqbnR*xS3wY~x!9TXlGng- zoh$a0^%cv=xVHNfw&t~Sb*G;AU+1}z|BqI1-NgTAclXKRS@%D_%Lq|+>jvd=Q#l1#x2&TALRxJ@A9eviC_K?-8S|r!>v4@%6 zZ2d!mIGxX~j|(hY|GQ6o|Gy`X9^L2vx|8SX%Ae3q_CYaS;y8xD%9S{)~?)mF~Z|}+d`oD|E+yCW=q87v!(QR2k7RrkAPAG>r zcKttugz#Vm$6>Vt)7$qOmTBIwcI*odE64)4IN!}Hrh&W!kn+c?Lj9`B^yH4bl>ZY> zR=7KHm5F|Ka1ky?g$D7thryp-g-B0gLa1>zTrqm4Ri?k&Mfl zl{^`jv*G0)1fRuit(dT$SlN3YIRtwXzDC>9GjPZuf>?=d>!>zXGN(izo$vbbU9o*61p_!h|?5x+f)mF_wfXWIx zB8T4AH4< z!GtW@Z&{kv5G!(BM7{dg*acntNBn(Fy8q|uBW`y7=T*Ey4ESaG-`#`m{mURQ>8oAwowe&!E{%SGTc=mh>3fnxboySj{Y*Ryc@!OlYMGf0p!{I zIkL^x$@|T!OqJ;`P%UFvNJeWH&-=}mGpD9?Y+Uis*dH4BYfsnv=a!$_0o#NKL695Jikl+@76GPhHkw-g!#!TaqAVye}%C1Rh?aC zIXh?R2(>y9AsYmtk3 zkHtq?r$@)HOeHZFc*6?mHS#>C*-U>`ejeh5xIO0EXCU5FC$T6!Sxn?tE##Ng%r6`F5SC?gDNXrT3iT(vz?X(f!uEpN}v*J~yCvO96(9ARI3+TK66^xTO5D?^zjXTW9Wzuo;q|NM7& zpa1u69*_U45$&<~)U01>hyD;ptgmX}-$VbbdePGSq@m*r*HJpFzN51&&lujgd!b%J z)lH2qI`Z2*5W#H$sJV%q;n*&iU^F->e|EJ+!pomy@K^W z^tp}wpUUiX*FFDr@;~oC+26nS|GSgN(*G{f>Uwzyy3hZ7v*$MW|1uxI zCH((j_wjCr{~zAxf4-Ba-~VrR9p}H*Dq0O)2EOrBT!`bi1*Knn|cl&OD>;?{awkO>J;(Pvgt910M_`c1$j&I*(ISabI zi4Qk&r(Dj(b-lS}cjfXlyLEr3PV-BAm6l<)zzb@%Y10!a(w0J8>mzAl z{K^XziI-u@rS1KP4>DTYhYvqn_%vo87!e8SLzY@4YF6cR3>FU#Us(Z(>vhBr=ys#7 zh>iNXo$2~E|Hr#S{l`5wI{#g=y2-{guga9RP07bJXMtnjvirXSU;p=L_i*q2{_jp6 z+x~Njou(RMC+^o^n`|j2qo%`gnI4(xAkU8-BhDZ=c?$e{2+|+ovauU*RG!Q_dvBPU zH5-?+87oZMU;EA4vbeG-kr#ka-AXTmE@nzj*B>wxw9~;p*(Tfz0jVzYoTYEO$yx#* zZ&xf`hLcoS1`xt8>q~gy7H9Mm*e=>Zu+6)>=<_Tqx)|tp*Tw)P73tLCT07CI;zMcj zglqi;WwWw4VK?_NS+Ub?mD0w^U1PtwhBp59EGk`N|4*hYpJnGoS+mvJShD}`9_;!4 zACDd%9^CK$ckyhHW7;&VE;wmQf}=`4PFX?D+AIf?>WYpp=sDv_Bu&qGTY()1;Zv69 zM4T;9NtPAoJ8(*EvP%Y$iLCduNTUr>Fq!{$tE$-~`@~XQvj1y4A&^P8ETE=9iBzm6 zc~-C_N?x3NIBCk7MH}RKIh&OQ`TO$|l4doJlJl$?!oP@ql$`y$9>Txs#q@kA{!u^q zWid2>&gl4}t;i(H8IK+&{9_e8OwQ;<^e|~=RrK)xL>uJqw9d+wlUFZZ@+grW3sI7# zj1I9}UHbmXfnMdHCkl|0Wl#E?CZY$j3Ts8di|3 zAb%}!!B=^Eehw~WSeuN6gf(u;nkP{Nd%Q1%^@XrTPXt)itLo_HRMp}+r@FN2EHGfRq%%9Ie9K{ ziaMRn$QJjxEyBIVe!v2$*{r-|oQOhVN}6dbny$)%6%8kp^KzWgMs2m= zmj+7;D&4QnYYO5;U-zK^yU~!7IF=Z7Cba~=yiZte zd5<>8qvVs2p&lonqUfZdbt8H=smmF;lvm-8lig%@2S|M?5Y=4uR1QeSZ3bfvr#a-O zQu$yocW-V1|8ZH2S=I1idCBU_jC~x+fbFrE0da}${=+c{KrsRwZYSj3l+|n^M8J93 zk}17pqDMGI04KvZ&rCT`!eBM!f^v}xlb33rT})tJ zmlee$#)y1h^5$LnT&b-P?8b(4TmyC=MZgoD!z9T5t=1keCGr=X<8pJ#it`?Jkrho{ z4q@ef0M@ZFHt^zR%5blZam^^6BsQCbyekQ33|P3{mBhwaR%k5ceiOtQKyLv;Qfs#z$Q)%%|l?Oo7X z)fqN5W5XF0L>Q|2Lxp%uwF@Zj{xiW0a4GWS_juS3JpJeY`hx90SDNqo2RzLTfibQU z^u>CX>HnT|@_+3g?A`l+-^t_Ip^s@p^YUC73W%d(mZq#2k%3^IgJ2O@RhNqIZ2tKf z*-Q2hlihfi{E6%(kM_i$o589jACWvO+E3P3IVHvCF{_(wk_p}|--gRsH+l%$n$IzY za|lP z=|hQKLfAShrLMxP6OeXcMUan_lXF&xV~@Z=oI0E%26<7AFIX**i#EvO6P1x>)yl(j z6k{|!V^0nb4*ZnQy;NS<-vCMRrm z$!bAa-qch`)4Zv{aBnMRldM2}ZGb{QfLt`oij)sI0NI97uvh>orFkVwgcftf zU%B63IJsPADN70E0y9oZI0%s$tq|`Y3(>ZiHWY!Pk|wp621@3lb2b6sXspPPvI2W_ z#-uXUQn`tvGStK-UrorY&6}*snd%3GZej%-1X#^dvX!=QFc;u+T2%r;**y$`w7nzy zf%R3}!0=(sk+M!%jV-<@A(+VqiA8^F8PJ^)IOwTGxkJ7eKXGa0xf@{08cx9qF87lC z-DI~1qS9W*aazygx-E8OM69}IM&$j^(3Tn6 z5g7;xV}EZDZHV&%nrFb@%FvagPxjvK{=vTdAU9Q2j>yTce|FZB`H&S5f0>lLac7S%mi*1%_(QQnefNZRu&Arii*|#zX3iicLqZbCvRk@QRHA_KA$7b8}T2oE?M8Hr-j# z;9us7o9t+dci^U&w;=f-eHK*TERLePQbH6roaDYAh4P}1)D8|9Qt!N`XIu`owAA5h zJVp`5mt)!mM$f>OBdx2lY9VTnMf6Q>(t;CIE+-n2#pqdGmo+I{xOS_-WDTYy@*eT0 zWjn1zU*&(6al%Pj%CiqkRs z@+IMyV`VFz8sU` zQ&uwz@tX05@8GQem~lpOMlWH2i#E?A+j9gYBA|~LGC_*EWjl_%C}=e)>yNZfL0Zs? zIL)Z0VjNx_i|Tv(PmK&K=VX;XxYG*jf)H?K=m4x5hbMlGiATD;(H~eQ4SW#z&e2) zkV#I@Wjm7HOVCXT7d*1!;B+mZ5za^@B?e6WojZ{zGXD!vNJG^l|V0!B0~DpT5Mefj&YN=KxKNy#geSq z7(~^qP^(YbSg6g8&?*Cx&*)st1iW~L8k9`hMr<-tDDQDxi+V&W_1Y_c3|Z|4!%JEZ z>$VtTL|E?Kb-M9KhCqnHa_S&3fxMQ&z9xPAe0D(!U8_4iiQ;;O0F^}p#B88nkFDu# zc^X|XaT0Y%E*`wLqWO9ZKpDTBoV+F~!3iDlaooh!t(M@J7GmrsWm}+sm?-}utL640+7VL|hzzS4ECz|#80GBn zMHg7mi3Ee92poTSRTdyj39RI$71vFoqo-{1g3X=ZWBS}LvY~VVKguh^uHMB9MU044 znK)Fly5Wi1#BoXyY86IfEsOIuB?Qbxbqi2Ht_#&q%~mBl7UPjbY7)b?lEN^+ehPAB zHX)}j(c~OG)nZ?f`bA`WP(-zf?YQ(nQB9u*)&BX9DU(r=^;}eN6RWbxF4U938)sO@W=R7Rx6jIC3yz?w3V7CkTmBbS4(UZ{eCzoctr4KX^Asa)>A3 z=10O|-XPXGKye*z*D*Ran6tojuk5v@c^AA{lsd6xd_tNsXSKW{5r_~%CzGtmnz?Ec zbeA$@wKnq8*9k`Awg#-B=bbwQ^IhJ^kI4IhS9t)07?;&t@B#j6am%wzoVZvT=h<0J z>v??E^7$FuEJzgKdS)-u2Nrdl$cjdcj5u{pI{IKO*=O9EHCCW?J%i}DE;%>!SI5Hc zT$Mn;U0Y1luq|6zn*rkjHM`N{G8N279tN9M##_RzfRb@uwy9bZf_vBOqd=^x*(Cr! zF4Op38GkSK52+KDg4$WzkfLl9bll06HL3D*q2R-IyF&%CE;*+ zvSt+I7NvN%`ep!ca#FOjGgiYiVQ|j`y4~U&HN6ZWi(MWX1K~*Qe>@W#zFf#yUn%r#q3AuiVk6d#e@D1!6bl4E-_t;_a&Dv$5pW*|@BW}0Y@jaDw| z?EZB3>25^GO3X_Z3$QO(JSv3%CE|dXt7Ec#!?+>`TZ3h(-t=%R^*O|_;M?HmKZv2k zAM{!G{?|Bvf0(koVl_{iYROgblKbDshx?uUKYP3P_8)ihY>>CoumW5eTRIoq$?`VVkBP-JKBPS>5nkG6k zd>E;>TEi$=VlZyTSNvr5S4gFo#hdynJY0dYeG{LS3fC4bEtb*(4hoUw*_ zvBiwdj}CPuN-iuE9`(QCEv4HfT2=Y{Z-M(m)3FEo+M0}>5eQeQKEb`JDImVvxiNg0 z{j|~`66@yuK|kGWq5+N{MB6h~pTia5$2x16jktj?4L268kY0c)IHv`;LuZ?~+9Je? zi4dC*aE2*#?l)yIEV!(cGck0fx6(c3jjCmr4$TlBb2JA=uLl(-O%#+?VCdIxw?e*r zv9DLHfI8-i_A~jVEgLAR&K4tLCi5y7sf&z(H&|P>QkM+gsbn4U6(=kM^@SCxN#4uY zpsfcAP#y{vUtHw`_yj4$LUVX!p zZ+OBl$D?o30ogJ^0$f@qGA_WjSHz#XSfB~@&WkSVaUQ%U?44mqbwtA7`m}uf4UaYH z%|Lf27&v<2=RtS=ZLd7-5h*v;E-0YitcYhU>&8;y9;5d6F0s$8ReSN2$F!>aTsGlR zkojxE1s#Tv_;H2NMfTHt;p$ysu<<-}C)7fe)*+U&Y;Hs=FM?}$`*7_)dWiQ8kAYn{ z9QiKD=V52lF-*h$KHY?tDVNZ4-@Lz4$B#~WN*<_W8IbrsGU z>(aNsqJKSXAarkb`k@5M+Q|3omU|&CtY!5w$TIt=yNuB$#3Fh(oZ{#i+C8Zf#`wwV zM&BeL-5t5IeV~XVUkFhkfe1TtI+&T$4YMvK<4{u%Zo91qI^3e9jF!_^rkjn{f>4ew zg57op6mEhCWFSK;4pypVM|2#JEkJBw#dREPuLQ;l`8Xg06TWgaB&=YNgVl-ArR+08 zt4fSc;9IxZ{?QcEzh^v`j+Td#*fgOSM zWoTr#&LzWAB?ccIV>((DyiIx%~@tldg>xe?xo}4g3WJ&q19w^!RBP3jH7M~sDihg z2f5@>{lASN2($lidw=LLSi-=qKV@Lat|vOwEQP(QlBdBcnSeB;;@E35oWqBkd|Y@c4S6YN!nxmbAR zS@Fv5s2dPtvx2a)n?Jke9^!qfOkFD(Xc-D&&JsRAQ5MR|<)^o=A@s*k+JX(N&b=R;vu5yf-{k>L57!1z>CxPTx65mUm~b4& zH<*rp$%-^9&cEucEOREqE33J-pcjz}wQR4cwQ=SxMh(N+7tHfWdeX};q(px__Rxh* zX!zDjJGm-a>mqg2Ar+nKSq`f$z8;E*vx1BR5pd& zkBh9cFW3Wwo5JvN?~FB64c2j)p2!^mW0C*9X0Xa+P}SLGma}u){L-?9)uxu>-K*p0 zZlU8>FU*U&Espq4oXJFNVlN(%&9~z1Q8C~2qabv?-*CHA4h`Pu$~Ko&_IxrVtLu0s zT&-69v)cH)oNiZ6+Pb!?a!?x5T*@CdU4CeJ1GL%1w_u}dqHE2vVnlYeE6!%q@Yd8BKpI@;LGEFe9# zj7hrbIxmePmNFp>5KQ@UnN&8GcUSLuQ`cH89gpXeVf80y zfZORy5AHVD#ph$Eq|T=Pv?D@*qlfJ+(u1pGv%D(0wi)$8Z0&+ekPQiz;zwxQ!Mf>k|Acve0HAY_7 zd%XY>;P{u;@N|(?|(db_uARC~E> z*Z`HEW16P5fcxXyciW3l7Ii=(jm4JmtSfq|_U#}fO{qgY+it#RObNFEr(fED>)Id$ z(4%Ug*LJ8AAN`iII9DGR^>v{hBtXxF<-?cUHeRYk%rCMkzJxN@MsJy##%>r?F0nXK zJiA7ui)*9kohLLlIevOJWX*WUc@84JjFfEtOezSb>ejnp^Ci{d3l^?ArYkfWk7+E5 zIS_jK?Y&~frM=XV3(9qF@elB#_^vKzc908#@uhSoJMTS4Dd7!mTArXf<3?NoaTBb- zM{Gnwj1;W>7x!KRReY-pd67jh8j0Ef)t!Yn8nRD~O&3ttyeSKDtamnJ{SkCIWN&wO z*Qe7ho#~-laPo#U0lYi%itmbwahlZ@ikLtE#xyV+ud&?fZgNZ4=ePepL)SE-)!y#H zvA%6W3iL-R1~ynON_gd$$rLf+Q=0*etskGQnZU-=k_=85ts!}x z73~Ff)M^g;92xgSI(6f;8+4!uB8|v{tLHD@zWeaqtJj#x=1aFAIQ>1i`rFHYE-Z%p z5+ELRU|OZkx04@#di(t4vjdPLX1q==Z^iEGTK(zJsIM)nv4-)6#Q$~%4IBMJ{_DTwIDCg0Ihv9H8d|3} zfFhr=hU^7Fdi3gtc){i?0ql}y0hTm~XJ?WzIM%9)Tu!}fA02hJe`#(dHneC!t%YSEAmOqJ+cGh`dt`rzlvXe8P@&9`{_+P6z_~K z()?&C)w>k~wU$zTl^Q`2%UyRtvBOuMnn$NOU$L{&jZBkE18Ve%n&5ACPYn>vzLAf5xz!9TM(QASZoj+C< zh4Ef1AfVmCjiN&3*6tPe_JfMHb>-?9x8z0QwO_!y*jm=Yx{|wdtGGJi>PpZa-U^!f zI@fw!9^K#n-ocZjZ^{O{lRH{iwpwTxbGeJp-;lh+Q$w_))bAj5CK z;5C5NZCeX28P4C5@#NR1#>%FX_RH(oPHLk)c{W*YLwPG^g+^6yD`tf~;BI18*b#{T zLuBLIlK&QE1Y9QnJ$d5i|2sUmm;dhO`6}|CGXLvX^xYoIz6DFYB|Mw-80uk|C(Y8= zwwmj2_7;rh`hmMPb0k`#;#zRm8@}niWfM3#0hSuT1kc1Er1eeKwdrv(7k4L+Yp6uPre|PcR%>Tm;L)Uc!>4d2Zii2kECF8PYC0O_l zeLkZCO0K|emV{&c71&Ns#K>O$V{&(rtE=VktRRm8IrLxU8Hki4lPLvQT>8!gB9qL{ zE)aLcSDQ2^UBOi=AcHE%9>U$a+lW#fJ2q*l*15HN+j#a3B)9429g{6Q6Jq&*Y>o z#ml0*B$dceUwLeD&x0@*nU>+OSwAdXL9-8rA|Xk^lB~4}JgN zCyyW9$A7w$XG3mM=MCA)SwXh)vb<<3GFWsyI3NS-UTq+@kHKJjo9u#&xB;if2are8 z$*+MQAm}N69P~+kLs#Y-pbFmtA`jV4HjSg>S8{jZR>^EaWVAxr=vGsnG>~^(2EX1Q z|342er-;}@EqE$MTj-Yzk^EtYJYb(Pjv<>ROb4Sa_KPk8U2`xoyM<1d`TXq*Opyr5 z8;kRV979ZWA{QSyHc4rJR7q)}(ixkSwJzsoSxaE(miFx{g{HfI?%=rv{$KG*?E%-~ z|GS5~UH*S~fB$nQ&#%G%*SXi80TNy%5I z082~8p|L5w#N6;5Fy!d?mDZwaMypa(AD0;Ijt2(W(Zhs)tfGg>8NG-eCe2LzUDnxo^zi>g8|3e_ z&dQdPS1(@jD5>i5Uu@h&NtQA?#Nu`NuPC|X<1%H#ufDifc-CG2Z(kn0_~B(TOV?;) z@%rCC+?WPTu<+ zTZpWj^i9p!@O&Cqg5eDn=NtN`tfO|X?@4ONN2Mcql6=jBICk?V61EH2tJhWF7L zmnE$#`eU+_mMiC74ydqHY?W->zy?cBl~2>@3ZxAu;)lLTMFnw=3FI=PgevP4B14wQ zE>^4t4Tj95JEIqv(UNh{cAu(X(WgtgD^^U|HkQK`dd_mPl_e~ZY2C?bH~7oxb`-s{ zCazeBlJC?^uIbdUJgS-AL@Zl5b>3SGO{Rcv=EH?tabb;ZB4vpriZ(XLPbMRNY1995 zFWEo%?;S54FwK2(QhsX7${I%2%x;EcTCR2(7|s8h1VE?FTcDjJy4qNRC0C*z#a zLI@1HliOk*WeukB&Y3#Ee=mSZGU7xztIG>kKwutbN9s`c@^(B$;!fHcvgiT~+%kbT zK@);)!)CoD5iaC(Jl*lSutQigPPP+4DNQRWK5=r+8p4})e4+IQfY}bI5a)4OaL7hJ zWt!WFWr^OSAA!3mVgOvH$1-4)*l(+{5YrD>!=ky|l&E#yL8{0Fkmqvl1f~~8h(wl{ zHDc0FX+ANlU&&v1K+Hhs(f5Z6CEVk+@>#P8o$Cxrh0S#3vA|IHp8|5mJT4z6dGFSR+`b+|hgqSN207g7T{xOw2Y82v`Z& zH^eV>TM>OG$EpGHnY>_pTxZ~LPCk>DpXjX08TlN2Cb4*k{6BVoL4i{(ah#IR1h#FR z>s86DXAA-ypmfVp@>z;cAjt_B)+M)sf@!h~BLK|p+p@pP= zA%|==QipA|P63Ws{9>)%!g*wPmX~M485Qhe_@W%Q;8-Lu9?~jHX6c4&PZ&DCEIh7Q za=02^DX!Y*6$)$FFg39;nX?Ny633%d_gkWXU}VmwrX*s3sxswfO>5ypT5v!ZBB+J( zKh7<8N}}Ba&Rh^0dd5SVfQb<~!gXCvj3Jp676ca;r=B!LqCm_6mw;Nf1xfa$iU`6h zHeM#V>Os8Lz$#o}MZCX4>ljIjFF(88)|N$Tb!h7nHTZ&YgN#>JEg1V)UESp5r6x=X zUYrTC53|>?x)#Dw3yh#7j5eyYKaz^P^#RsQ`7o8fcB^{B+3m%6$>U%~P%G5Dln9l!Jv6q)?Q4S)d5h31TF8wXNrneETvJJWqv?;)?A8qm;dX zy?$Z)ex`DP$af{kz;}wrqA;=or&kko&=kvVr?aA@a zB-yFeV#{$|`wc>@T0`zD*PUp#rpzJ%SQurfJh?q8X!9hL7Z%RDz}?0QD@55b zot(VZ-rZz8W#fw_(_Dt3!{ev9L6Q=^5qHM45DWpVUL{|FJu$O(O^cC(wu_uP(WV8^ zObDH2l`$dJ=X0qs8#B&nJ(tn*Ac&OiXf+LfviW;U#tgq+~9G-Z>l5Hhp&Y=#t@JCOk`jAQ zPbT>?IRm?WHGYzX06&u~PHUhf7#eO!nKl@wuFy3*>?2R{@)RxMl$C6z)iz!re=#;| zBp;zQLuZE1FlKcpz<=)}^h!`sPsT zlk`<0$+KA|%`O~~&YEI|{e?i=v0M#Q zQ&8;1;{-YK*9#hzpz_Dkpviky+w#lRof>_fb6chChs<$oMd)}>QGPV0La;%pb*Gt+ z(n&qteY)$Qa=o!c1j9qw4a(CukOmO4TunLyf+EU@0yy zdlFm(!X>zOG$(8(1l4g_@TR74*8<*UoNVEi52G5mdHDe@Q)XF_@*xKx+dyo22Ro;; zTziT0gnU=lI2q)5B!tBFbn04IZgO&oAdzh{4%Pzl7Mc4! zH4=i@I4;;+Unk6HCH6VdDp(03xE)8S4*i4~1>Lf{jRrEZ^C>Iz%~xBc#!g0EvqMd| zJkVDL8vB|NR={S-zzaWZK_C#Ib6SDIS9S|y5pM5D{IR~iajU94Lj_1#r>suoB#EMj z@3LwH3t0$kLaar;*%e&>Bt)yv7i|6+#o<4rJUj;TC3M05nC0o1)+vl29XG7z4-*na zCyW`3S7Xe53pTF?*ftIYUf_e}0GL3S`DDpO$?w7V{r*{C{|`nKV5p>W6%L<+-OhqY zkKt#T|M%g+VaNXeKK|p~JP>L`M!Sm$p>>l@=onZI)6!BTGX}C@uDZ8m~IV&m& z!nT%Kj6v$c6{Y4pMp~*h+Fg-uM3w!BR+av_pX}`>dr@jkf<{C-N02TgvE|`Ou3;c! zpuDNJ{WxcrEZ4zJrlr6uFqY<`B*Z_W3pW2))+w0u<4f^}rb!fA*9=ii476w_qzM%m zFeYq9vwTDvT4%o`*;!4e+3bI2;&4<>8i9!pA_GPCo_@P)YRzcTviyJQmYpZ8)g@=* zg(;QN%Pjq$-B0N_nVHz(Y1yRX2>%X@hYGzH#n+%tGq2bP1qswxM|$=iBzuR+<6mJ< zU5o!)0@yljEav}@clVz7`5*T8A3wS0|9A0hXe&V+ToCcPL!U%uVV=OrQ3id*@S6~P z8-r|$B{IGMHil{EWmHYg#)h-3uFIOG0e%)@afCGVTyQt!XN2%69g%_H`Bj_eD$Dn) z$(yn{t{G=V6GfdcrR4q3(T3ig2Jk%=^<%uT8AK5pOplNwH@S?jTk%T8_ml^H6HuHG zWa5&9Z#4-9kcn}Fyer9+PPQ(d@|ZXCoNe!r(ygPGn$AC? z_Rd%yowZrs#90w-0HieIW6{4knK0V4^)>eE4T%v;W57o2&>O}SqX5&!I>XEWsu|Tg z@fWzh5#CTa@y}aY)1qOFCy{`ypkhQsXK_WUpChs-2>*Nvv2{>8pw+Ov5%S9U96i=; zkYL1sSDk4RZIGHx3;`DT4ty-@i@c;Mypqn(d}!JnZm33q7Dic-aWZ&YLWrH8KnF1p zBVV!F)}qM5;7k9atx@x#U4S;7`kQc`&f&JFB+bW?)NN6q+ZXy%BDbc5Ps_RyO=V>Y zs+erX1W(Bs*?i0JE*ha3v#^vtnGP5S$>6^YT2NWI01u9CFW~25UEpE7L6~f5s!Z^;f;t+vyeVfo zyyzD|omN#)Ru`a^UeuVEV+)b`tqN#lEcUO_m>OFrs7`oktVOkI)b?sX89=%Ds^c$j z==cJ>gg6?eW7V_rQzYTbIIogN22#UB^%))GzWHh*Hz~R3{|f|+w|_Z$Zc&WLuTq?Y ziYM4bDwZ-s4+czA2u3NQPvQfK3BSltK?=VCkoXmovsUnC*cMAvpxPF5cwtcH>VGSX%(^u;3!QyH^|RnYQI$jw`uZ>2@WnG3@vEdra)a&XNBOC|7wdd*jB;xLy$zl zL>YzrAjOuk<6S1ulCE1gyWB));YfaipQW&7=RO;l3D3ALArqgyALJQt20LW%kv8Kg zd*L1nz6*432Qqt*4=_wU{v8-u_tEyIaX zV>uJQ=VFq`CbknP1?c-PA@qpp4h|nCCIZKKjh2aGc{mjsfi;+tCy~HYEK-QG8jy@s z$^%S~wDf#sdMpFMinG*iviCT_%pB%4wdg!02I(4mv1}aL?(D=(&f{@bO<5iDHftCU zY{S66Z?F-c38MUa=uu*$<@B7Xsb%(ALN+$^t)VuKlO7#P*wR*NSR9TP++4$<@1`M9 zDb0dlAC{hKTgPR(?dMiOaBr8$sSV{9AethX0{}8MUeZy-e&a13ISDvPn4lZ*oj2q8oiZ(pTT|ATLv2j`OaaOim0v}IHu`<=| zb<4Fgvx<~bi~SdFxX5XC*sPbA&sYJw;Q zu*XVbyM&j(I&w4g;J$exU9li^J1TN~!!zTYF3pH{bfgv0uv&7C;0D?z=|rsLMh`3s zafuR@#L7_?eo{irxRe;F7H8C*Kx%0kCT*UhSSW*mD(yIce#<7TW(B%2i;Y*g&P!(s zX)+sj(z~C754_u@gUF@fqX(g@`TRlzsmx)D0abz5K*-m3a)eBW+@v=4xnQG2c_xOJuN;bEF za*%059E4TV5=W!Ax% z5YT*=%~;tsMktqFy!L|7R@9}6bR`ZC`!*Mn%U|wISnB(xsrH@gi;$K>e3+I+S*w+4 z%Hwh9ErV_dnOHB|I`EEz2d#~_5FE7$JGv`L|4_hOsZ8Yhfm)tJpu-l>*QP0@H{i;E z6B$tesuzrv~Q11A*DoqZ^`zFX7+Gpz7*NvCq_X>NVDKAC~B@M(c zme&#Z<$}#eMEq-BH#x`iuvSs26HErd6+6Me<~?_OWF~}1BXvY897nX;T!sh>g$;`~ zHloT8vOEfGU}{(;`c;|QMbzipKxueB9(V3)vvDokSH* zXkJZe;x+-tF_s?Jr8rJw<|eZFaC03P@-$_^7?Dl9-c{ceJVj6g@QgJp!K;OwKVt@9 zWXmCcmpPkJ^JA-iR3y!6(VUKCDKqIhlF$Zm$|REt)5c37|fFV>c=5jk7#!XRy%0;2eHJ`TgN@_~F+=R7bX*3?73FmEg38 zfV~a?v1pZnvZ~fw0U(1l)EQ2cJgTx2DCnjI!&6gJzAPH&P>(mXZWE|QV&^sET||5duO`7Veo9=q z&g7Iw^57NBTb%du2FGFC%T-Ues(AXEYNtg?rzI+<kSq5g zjlet4x#iZH7Gv4Ov`DQK$-~G=sVo`!yRs_t@_bHs1wxzcGjKzc%oToB;y-aCTZ^%^ z*lJCnhaXSi+jH|t-8UQ=DMU($PJAuPIDktt56+sXdzHBiP*tWvQaTFVfdVOnJXdEW ztdpE_{zhKTfhkA^*I&^Fsr(|!dS$g$UbjVT86n!MK0O->1YcylZh^i0rA^PJ@G1l9 z)0nf0qRL1)z_^}A@~TCr6M#~JOVt@iw*vNwjvHAE1#rRKhGk7^cFsPj`~X~Cz5pix zFoFlBteE4Envq%Cpvsti8s{y~E?E*G;3HMZ+$t@`UaFc!4+?Ep@-=(#LKa)r0c%sR zEHUt=8eDRmgRi{1t-~j zW&(FUdw=ni#{ehZ+cAF^J4urKyd#$0v-h;BeipyCH$~5mRJh&Ehnwo6O8Y@otTtWA z3f@QqA8A}s%ryzlS{ayfN<~*VyJR&}F^wUfdN$L`lso7nRu6yP+*XqXg(i2>qqMa` zR~Jq%Yff1#Z`GM~XA{fD?Up8&0eRCI=J4W%(V0EDx?gW^u31#MQw! z)0)hlY+Wa1=LMWv@VZMTR~8%Pj8^hzT^9c;&+td8qBzQ*T4ccAI@%)qg7`FIzUNh$ zf>q;~=6J^ytFscrU%CM>50Zaov#dd9Mfg2y8!*iI$j+u^dErQGT&J$ZxrjgUq*%$& z@%XMxd{Y}QO*t@gO{pU)G+BKRB@=pQ)bPYk%N}Z(uBGM$9U)jqmR3B z0ukU3U=S*JrvQNgt+6GClY_(#rhb7lcA&AIjvb{b*^*XE+h8H(1?UjmdH7L>XK*2N zqP(Y72o#BF3D=~Fu^U3;2BRXCp(RaVp-#ANq#7=NFmO|CI@^?tZNFafAP&j-w zQIJhIPiND@I%j1w#j98{WA(Y(S{1P^kUlhR^%vfqx6sCao0K&HuW7a0xnaqTM0*?E zHeb}Sqfc%#cfttS&>72Q z)U!p`RicfJs1xKHtNUgxzoJpguW8cq>l(EDD&{P8(U3dN5p1T zGbdYMW@-o6sI3Np0csau+!gJ1wx!>el3Y;7Au~2wfD*ZY^b78tHAiwEkF7GXb;xk7 zO%2o2G5q_60vudWTujtlMXS`R#aN}VaHXju2$ktu%QE@rj)7J-o|^NMob{G=v?Oc?(&wP77$c5B@~v?#(6dEGx1Z%`L+gbqr83fMo8kGe&5h zUE(_U`|%M$|0oRo*fe5t!&G3G*;pE@143Ex)^Z~An^)#{W)`LrVH9apKo@i^gL`z$ z+*3C)PSyel>Q>Z(lcKv9_{?MzCQ~1lX+n;bVY8fJqSoNY$z-rY2Cs^&$!ISARNhzv z-2VoGpA3|#Cbs)SDHy)^AUKy}S1Ew1fORxYU9(Cwps)gGL7cB>gPbs?HX?Gy^73QU zF`e8X0lP`u45AI9?I!U}jMWWxOAzx=T7Z#-j6TYY5CM(C46dIf3X5=iDUJ?gMvH8s zB9cr0PRg|tEqZDFpbA-B&WXyP99A!~A|1(_*&k@7E;Ly~Q`%5vM_y|Fx z^0Qr08negnjtpQb4b+vnw2{R{&Pi7MOA7e5Ex#_7m|M93@2=B0E3(FR*&UIK>|=Ib zXXy|ejx5=`TeEl5n(=9wrz7$xB5;B7S0>Hk#gFe9wYE89_aweW1Ui(_`~}PD{6w7eQ$8XG z5s?m?`qiG0Pk0NaL{^hx{xoP4X4S?EF`8y>7i#tD_t1y09sA(^Su_5JviIi0H$T36 zd6G1rnpN6Z68~wxoB#RI!R~$hr#pE9W`K}pfoHJ8ay#)=4yxV0{2bDbNQfrt6v6dL zlb(TYead7~Mrl)pM;4wy?mFdyr%$IH53lIvuGTt#`G#!Hmm1a2*qOT~aB`U%uS$?!SxYn*0BUDa$J%o;TI4;4S{&4(4Vadsq`9 ziaiihSbrg-XJkv6iu89n*|s2L6CVhE!>>)9%}(0MB>O}L!$G(t;12OAtco5noZwS- zy-@!*$Um|o6$k1QR7MMu3?D}7tx9_h8eW>F#=HiCQBTSmgI(es|4t-|fNDj)w&Q2Z zEXH77NybVODo%@On=4Dkk~14(^l<186MO~hRLfA5<4DEm4Q+gO|UwG#~WO77Gn!|L)9j4E+2YEvU)hPkYJU<9K&KlINCX zaq>44XFK_8rY>LrPQVGm#1?fX!oot~SwDP8F0!J&sHB`3r|fHgp$~qObxEi+qDri`$zl@A9&EP-F2qi!nLKQqQ7do z`7OnOuQa}HJb*WSBkdMTq=}B%E%fl1y=$@9zTF2J`sEYaYh)n*7_0=Omzqa9%|h4M z29JqWFn`GL*5*wXn?jb!97Z#moq_R>3=+L67n8?kXG6EAwqvys+@kwRh4=MtLf<93 zxjpwR{sMU)VJc13a2@F|;9=Z#SZUGQPL85cayKIG&J3mW zx=V<&6E7QpcT6W6JAN?7U#AXQ2_caEvXRX8&Nz2(3!_e`m*xk3uFG9@80b$+PmxD zS_}F&Ha4p){jYu9asPPFHShm0K#c7tIPXsjwAB7**T4UJvU~9O{{HVS9?2GIksAG$ z6AnpvhHbH5dP>NENdue{gKa#k8VguEkWnP9U<2WdY)e>gW42wlL1g!`))CS@kK1)V zlOiipRy5>sN94CnZe_y@j^nvjx_l-sNPVhU!Mpo^uXt>f1UDXL& zr7NllV@0bH0FKLA#XF9{_{#!?={ow1u`Ep+kq;PKjmvyQ-aS9I$Ia14Deot}IyHn~ zlcv1m{6s!$70}b&r+cfTV>b5tef@X-TzCIRlL9M=$AYYY%ksbO@B8^*4-X&R=l{Et zC&0zQd0t*RI`7NQj!k;A?KDEVTyAdT-!#4;nW*oCI}gPSo+PKFyC84**Irn|eTy zfrJ6Jkw`kT2lo@nADm8@XI(K6-q|8#Z$+;?e1Y~#>Kfk-(x3}^5g4rMmbFbs*BWSX z90c+}4jHHVnpD^6l=oyJV|JacU_pcl+)pjo@lE=4wFQ=Jt%eQ*b5kAN3nM zlT2*ep4isJwr$(CZQJ(5wr$(mC%^xB&&7Lp`m$?Rb#+zmex6=yeLw5+SVb|TX5FQ0 zUsUxKS!E^)RSk`|s*MFlsM+)LAQHhg#>cIkPuw@-=G=j;K?mtEF0LYO+7s@sU{%TB zsN{^z*}?1aQ`HQ?@N#X?{usmmF^_Jd2|Y#jO~MPyRfb~~{mae$ zAL<|#3U?bMt7WG~kS^jre7u%mlgR2OQnr%QI3Apm(*zwz$G*lDUmcNUOwRA#qe?ti zm}_)F-4P|RH1BmR<*OuLyOlz_S)F##VXtH zVGS@XYlHsR(LJadS`37tIgKfQ_)ypAS9098X3M^&xRB-GMwUS+Ao8v6^?0&UtiwkW z7fcaa7LS{*e3{iWi}hePbU=S2PqZVzEAFq=Rb}iADe?wc2vwhHl-9=#?zNT%r>cdm z&OATo^3zOq=LmY7I0WPQvQ%+B0vQPGoqaTLDRLjCXlr~Bq$HN~H$%g`(=B zVE64Xdwd|+Dk_@Qo7y`ci?r~%v0@PIwX_l+hs|$rN9D_-7UFqK& zUqvV4Gnk2L@;J3M;+o1;I`jH8*C5cMK9|`@{_}L~^g2fL1f^N91=u?))l6EoDP*={%8p+^*fuA9Ydsx3riPOgb0i9Xz`F|9hIY?j#tKq{b#yj z*#|8!g4b27sNHk~BWYL~VQU+tk=m=C3bE_fF7rbWpPL_jkHiKq9&i3JPD!IWlLKlH zXG5m-?+ca&eum>k)rg3mZ)XfC~5InA_!D zW6jDe_syA-|Qx$^>TrZedbsYO@^q80`3;?=g`MNwJ! zmJdW9kgX)6c^0}>mSSga;9coF*YjpxSN1Y3xBzb=!Rr1#WeT}|SfBN`)}HZ z)13n~rv1LojaEIl-Pv}0 zk}r|&T;}Ye#^7N)Ykh8@m#59sAL>#xAQi6W@91{p>~~QWT8jC`9$cie%kH-vub7;8 zc#|MyKYD1`m62gKeK@B@!rJz6yU=^7+>E}nMmIaDv(?TlZ&J}OaF*A2YLGsHX=z3D za>Z3#X*we8hZw)y#Dt>GDaG;$*HZ!fZp?OZa<7%8Q-nDiyGY_12FTJm&ZDZ9k z{JG6=fM~AeJ3z07Bb&gs9(k4t(A(Xuy#w%kx%>Vf5?59AUT~z|XGsEns37`)UEx?W z0zn+M`U42D_vIwxV(0Rmar{>#dtbwo@9k-Q?>9crcISVK63w51`+wYKBf8x6Tbn=5 z9zk!1E`CS(m)e}J#WNBL+6RT*N)*Tmb=sytke~7D@(*OKKY}CQ zo$jDLzOh{)2GXF57OsyBn1-wxmEM{#r6G%xgU(tj8KKn%)7!=hQJSy7A}P2B-tPiD z6+{-L^@wrv($dKB{7O_-a|`G3f`dw<`>K%#-c=f*<)M&D@ekGBz+J0kEjQ#PXSVxn zHgs>qj#uddNTcgR#2Muy%t`j=6O!*I*(1@)CZO`sD!32-UL6f!ZMzZ+csSK_eenVO ze3(`sBkzg~U$h`;9_y|@X0bB$aJFT&yWhmb=#EWgk?$zVLa;xCrh5K6pvw|CA<2*I zd@BtQnN|~exg-so7s<=!6LyMEF7%-hr*8BZ$mb6Wou-+6y-NJhv-bVp?n8PB{^Oo` zx8Y%D+eZxGX@?Iuv&rdun~4Fmlk!@ryFl19Y)7bSxHPAQGUeOF#%&l`E`5Qvb*~C* zi3eWE+S%I%&UOZ9-myIgZRlNWZ7mZobqExp37xWCgC|#Ayx-afcPu#^0x zo`N)Yw2!l(pJFUD4W1{g88ZDdryEr{{eO+=&Y9md$shSsg*|GgFi$2|kA7sbk8Sd; zxYu+79;jLj{B};iU2o;(+FKY~ATABFhMhzxPQk=Fw`1p~x2yM%(g~D6Jm8_EKFeqbA$$nfU$k)U z^pDfd*n=1!;ExpN_fSK0w82A(eeJShBc6F@ z#h0tmDFXfnsO|1LfqzCg6*(tdAGUjf70 z%RWH2<8+X1r0)v$@20_H1sNC$Z-Z*^s(^M_wo*vj$$F=6%OOCt>+u`}k@^Ur&-HES zh7WKipULQd)9tty;`#or?9{zxc`m{;QrTj}jsFV(?Wn;88?Pt~+FK|XEwoo>IM`pV zzAgQfYz9}YT`D6^&dgN(b3JbVbTPcVbXCi_iI33g-f{$>j^-V1%mN_9%HpxU0Sd`q zJ^?m4vC0CWp#4j_j{EoQAU=7vl@&${9m5YnqKx*sO6q0E!-5J$Mw3vaaT@7j5`E&C zVNh}C?uYnqV!RxIO&ug@^}7*Sy`0`}2h?Ste19E93yczu9Z`I>FTV6)hh;6Osh*TMhuhxP@i_cGn`<^e_3&6inIn(=`*D^qW*(MRjVp{o zD?4ikh@G@MJd{&RmN=!*R>mX}xnY@TgD z%Nk>eT%M`fe`V?G*l*nI=&4?nyi&!h8(xEF_;~j!2-Ru?g5N?dSH096Pc+=UXq~@b zwiSphLkzrQ)3$LX%7!geUk65pIeokCjz*?7ZX%U);a^4&?ldvoD7onZRD>(75JnIj z4YtdJx)R8;HKO`c131`lcZr!d!}7XCbWbY~9lUKg=3M{rfr+5t5x5jg6g-Rp)|4%H zSog?=P>W(-i$6KM5H34cNZ(qsfZ$1}ckbHz7EH$mAq9LZhB7MRX8WXxQx~feNUlHz zN;{CL_isID_Gr9Tf4I)CW@%Utsj$f$w$PwTBbkjrg3eVFD;D}~8$0WQO(!SyMJMEf zD)t;-8t5SXO4(+w!E~8b_i9z0!V(!^rSY@&1V%_J~ zIsNkHbu9rvm&}ti{+?8&)hFnJ zaZEYYgq?S@+p~YF`~^0CbLh=R7)W>n{qvW)Y&7F$V7Y^OLFCH?zofx|HeG6v{_w$c zg>k;p^Qr*-N(n#nP_TYTzWi7MYq^59ytmqW1^bVkW}k6zw3K%qTrU%Pr0En~FPaXU zl<*Emv`mM3g1)Hl=YmI??wc|3nUvVROEa9k?)Hlv*v=o3R*3kOSnCW^FM6-cIwAfZNxxg9FdEA6>nhEmk$`IQB*f8PhKX&-?4RV<946Zq1@jslo9IXh>|a%rkO1is2JpUC1vYKZ z0S;HpsEWcT)8O^GaQ=s%h#T#ja0GWo?Wj=a^r$*kk2UmDmRMMmnj@WUuK z^9B%`>=@}V4G(EI_W78yW^FW1Am_gU5Ah3({(xc?^~USXdTIsy8qn19fl2>*U^UV0 zRr)wGl~Bf&2e=E3=}L4qNthJfg*Fg{?s81W$Rn=UQZo;fE?L=*D@iW-o9Hx7`U>S|lTIJ+54 z99Ma!s|pqq%$6rg#%ntb=~YYT`ikRx5B&j^8PR+i4N=Efy|_@pkxm4BBmZ>-{B6s) z=zqpVlP35Dvu#fqLo9w-w#_2i9G`5vu-l!9&`>p9TPV_qY$aoxqG;}!lgq_Po07zT zZTmY+CI{;(T00`t({-)6&3-#}(dtH5>RLy{Rn%(d8BUr_-?#q!+k0XSK-qx46@ZNw&##3YDOdpT2%7=3-2c5)|mUyXWB>Rxc3XZrOw zY-Vb$uR44)W?G(i0xl{$wkwPXDnES#TfSaid%jP$xbZjM09E}^Ep59cns!rm+L4TH zqO!i+6Lpge`a0r%tAs1s_Tj&_rPb%YlT70B_ns;Lu5`v00rzK);sAhM@vq-ra^Kca z0lF@jMyNk!59Z6h-uH)NV?6(DKdEnhH=LvSMi}iPx$rvq`@NY~K1>%(rM`Wq?ulTq zC~Z2N{FGn#AV#V`cVe#}e0Qb{w*i{**&CytN_Wn!s>wvk!uBa4rKs|!mpjo51MAt` zWZxziXMcYS8>C!Rr<-4nPN>ICpU^<7PbahJcGp7?(M0mgRREPJj?4tIP?{yxeM*y+ z%&inHA@Esi$k#QXS2K!a(-15D>4vW?IU0F_S5N_HPHD8l5MeHv0Ymzhr(d#NS#j+i zKmI8*o|IOt@i8STH9yr_+xaEW9-F|H^uC2Ar}9-Dg4CU_fK*w^gl6Had(5o!@hb#r zk*?_2s~^TP4ywhYyzzN$vT?MP%~Z?Mri2OBoUrwek6F)JobG0%h3~g1@!L#py&IlP zB8zEeRK4IgHa8xx&8V-K*vnh)&G8zsX%mH7j-5gg*h zWqfO6m=L&dS;mYhP0SdJ>VY(p+_FOY=WsJnR=L=zm82qC;tgz^IAPiPL%5B?q`sm0wj*L4xoj8Ul zp^=7^1unYtJx4BfoP-KZ*x%@5F4a!dT=(&ia1Oh>V3S1DDGLETD&FA>Hm<0*!LpK9#XjVO=u4p&NAqV0&?cyG>hJiYWU-Zx|@Zy+p z9O`PVjP}2Jhx?8r&V`v0j$qJM%lBEoE4#HR8`T-`jq02+yf?;-3b(xH^QIkHBt9>aO!@^6lrm_MA(d z^4Vz3UNJQDbY!wd?DAkp@8~40B^lY(=%S>GkKuDv%66Yagirs2T|jbn*oNBGQ#>C# zNfzT_ZGRM*B3kLf6sCnQPU%5U&<58%6HICXj!H4Nw?KGN8UjS?`8m>FVf;79(03ajL$nGh&8Mn=Qu0>PukaSu0%69@ zorl0qYu~R5x=Q{{j`AoZQunBNrBzbra}Jo0SR5GAI>|2S{k8 z?luE8PS#A9SQ7q1&%9tjZ2FQ__g4blD5;->Vci@r8Wt>mRqef#=^e$qm?=-Q(DmL? z<@ZohWxP;+^0ni#1H_i8e#z$?z3-T_`{p~U1H_RzjotCnj@O5Jm%{csvEn$SN4CSY2_!=^4L%9%tcRMOT>q*=y|EuJ@4)F-H^1|dQsdStrs9&l$l6qx z&!DFo&S(7j9%Fgl#^RBg@D6`x9TPkB!^n zfB^Lphy%Nb;IG^4%8E;_JT-l{o$_hu4I5_$=q$nm4v%7p?|8m0(;C3?4B97t@C^Jf zzoSpz*S2iH!`$7S_jfFp`}3$&`kKHwir~A&y@2&w><{}Izm%`%u8zsDU$8DS5`tbG zaUm<-GDA#o{-goVNR-2rFg$@s{khm zK8Kop-=y4%tZqp^>IAsm^v}FUxyqJ=z0P7WX5n5M5691em&5JJ#_jQZX@&^)AM=_e z?=LdL^{*zopzx+W(i|-$p%XQH zea3-0hw1GwN90}`uZzog#jt&}(lPK%-9|0F6m$;Mab$XjwhL#V@*8#WVOZ=CA25Ay zJW|3N$I~LnO1(?pgz-t3P1olvp8M|EOfv@1CvM3wdy8@$xUT9q{ihia)Bj{I%rNEPGL-oi7Y3%}vDnFwhTXr<|UG;X`>iudR}HeT?mAc6N`i;@iC^BR zjoJ1wVe@Wa)l0qJ`r}OYy(AnjxY&t`1P+SL!EY95?oak|a;h+H>0yUtem$|8?UrCp zon?LCA3mwF{s}BREiW31I!YMLgf_}}S+s@?SYj55Kf9MP<&$!o^2JsTTFEqs=B4Uj|Kzx6Wa}5?p>D6O95TF}9^he&5%9GVO}m;mK4>)Z+z zy=R6oBQblx!)7JfU#6Xb@e_f3hLIwdjY_$wC8C4ZkeT}kJs=i=TZ1k~S&%L7J*B^T zxjhczzk>RPZH>;3ab-izkX#mG;%!6G7}!CtTOU!jJS z7al{f-O15rf6qb+oy2obA6HbLmh(`mi}!>2U;_TgWz)BJxAlnFchhS#9E2j7vM0H8 zOgIJxWuQMa_HBGnz970mmhTob*3mcjKv=6Se&s=)r&xp$njx$MWePw0X&_(gzU&YO z-!GO{yFowKQ8w87*<)0Qk7eo9j|U=B(V3%M>D;)9+0jgBQcr;sV@*U+o8n*>-Sx*- zr4@5}xWr~LV>xhO5*U)s!YT`k`>uNy@By>hD3F~FhqtXJaKQ2_MI$wT9O{cL>Wi77 z0gs zon3Nk;fSJRsB6-!v$TbD~VhgBtb;5+{C@#+c}I_ z;NT~XuVFdh$sNyBn1SkH)c4DlAc zvj4Ah$|{-7ku;V?G`Gd58e-`?{D>3WFsX#v#CZd~1-{H!_>%kD>TeB}4v+D%vo%W& zl)*ELdN08$YDj7ofbrTGv|xd3xDqhj5R$-yYn;!le$>*uJ8h@G)-jL4B`C@Y;U!3f zZlo*IEdTWSZ5PxwjfQp_6EC3%7PGo03J_tCY@X4fAUt%-M7&dHWgWqlYz2vpVLuF~ z^;=xUjVjs^Rr>22| z#c&GB!EvZ+$N2_G%Q4=4h@~*^Z{%h66%^q>?xU`OUIo~VCMqufPtyRot9Vay9a;?Z zy!61F;(R?(1xy;rbTrIZ5Xk^`TA^lgu$SqzXds%2HC88|l#uO>#Seij{CP4O@Me4U zI1*Z=$VQgI@XxsII?EJ%s_0S_D(f!9dr#215K=J>jloEcHf4UO;*gqfPTEsS%#sZQS$s&?ysC*ID*RtIt}T; z-u=h^iLEF|#d3&-#blXy$E+Ny5HwO#C4q+Jxz=#7K8`y(Txq^DGicvCRIq)an32pW zj=iHP?E$KxT`R2oSug_@<=6rvb6L*>(itG`GwmSu!VYSEO69?l&?ZV zgkS>V6oojP1<;%BakcQ@lxcPB3kOPO2mVan_j1 zHz4wsEEdKDMCXYf^iRrF-1INyEwaJ`v>chk^DsnWt`JkH^38kOag$X$ zjoXvHXPVB83q}*~AREMiYv)FW@hJ9=#L3d1O+d)cdWhkT+;PZns%O5dKuUbZn!zH0 zj*Zf?7SGAZ%8u|3%|!`UAH&ZQ%S(&z`;RC3-6DSj%w#jge&caPGxPFynTHNN13#dN zEGy@8IT6AYp}N-<+S}@Ky|Q$(P7dS2wzdCAqwkCY@!*y} z#N!)cI*twx`XpL+?@(ZDF3@tE+hw8VJE(EM?%lc!4)zGv+YiNh8LkQaBdqahf>l) zO&=wx;f5T73(qpBwv?EG=){@j!4#OKM0O6R-<7>KOpab^TSGuh=d!BF99}98kY0^^ z$H-OnQndWa2&s*7x)~2aR0iMm`>2`WGe^EUnaaq$rswMQ63XFDeGav%l*GZaL?(}I zgF*9+O`}P3%w=WBNxuh=RhVifU z4T7f}X(Ct0nYx^=SR`-@om>N)Lu32~FPHYkUZ(|rEd(_%G3&~u6}?N`Wih{40opYF*2>&GRjy=bW^Gpk%B#l3qx8GUEHfVB2_J}Z*O#4@e>{U9 za={%YOb=`<1JX2z(jPXxhgHwUs*QGGY>WbFdhpHF8++qVpkzbVg( zixU^ZtMgj>3JsXq#mh{CJNn)3bSzH=#a?)SOW=A%Q@!C&or82uHef`Bu+&&UTXcP@-GkUQ#@ZS`i*~gh z#JeX&jq>Xrr7!G=9vV|`~0Mn13vSJGhID_4IwW2HV+Adc~F z+pVv8oX>B6=q-3*9(|mH#}Uw)?1K*lAoGo?o@cPqnwZ9Q;HcEzi+&HPmECBSIQ&}j zCP{|a-`|JV5gzDdzE9ZyZxgIeL|kP@eK{80Bzg4N))eti_Ng}}i@)+702dNy@>P0} z`0;7$dhdn%^*34n?6m^M{12B|vMM93&b^p(WB(=$7q>924qywGA?Vb5$Y)}{OYbeL zjY+9Yh2O$@F})R^Ml9wozUSW@9bT~_KKtlwq~5d1MtRPtBo9(wX1yHlTyOQ#lJP4gn^SWb&DRNJZk zHrknAGg@O8U7HEUlY-e)2(@4H5B(Y%na&cAvijwxNN#AS(BK)=hX}jnLd&MCyM;n3 zBHZrZaaEMzVt-jJ~eNJzd< zue;}e7mYesf1s4^#<_jIcXuy8PXJ%B2tN=0+0uKfTqo7h5(Q}qr}A2sImU6pPVUUk zbBaGr4{-P*ve>{w%W3)WpTszn@@Y03od;ZI(oGGzuX&J~n)a4o*lNv_>(b8iV7-Tv zkxQ`?#~q8m4Ej&zpIyjXp;ImP?kXuBYVxO+cdMBe(m(^phLEv_9%z}gt>6uWa%X1; zUl$&DsRMXV7)M{2yj`FE@&y`oJl)i+8Vs3kqc-|t&9Xu~ft-h%G~;m|Z@CR6SUOe` zqbGfA?&8};?tgG+XG+jP9jDG0gJIt=O8uzP@(%b@n63LTP?>t*GJT?h!{UVNzHgV# zr4hExo;qm1qXY^}LdQ)B9|fN^_4mOiS$~W_xeLqC`tRRX97gOIa2amv@v;aDAX6V= z_400>N2oif_#XO6*aXu<XMtVqK*_grbeFr$gG-vJYvT@ z@u?m0mzyf|AQHs)y9be~C$@ZEfOq)itG7)XsMK#fug_XlfZ96T8|kCl`uT#H_Ls}u z1N{lT4v()LLwJH*0$blEIiB#GG!zLxxed1DeYjn}F?T-OBnEtVYMlNIhSSATV2>YO~H4|=S0(gRSmko@{cN$ z({xW@E5np`Tvh4|&ADqEoCrbYagvdl@eAmHlXM!rnzueZ=i){1Dj4&g{42xGQ|iXL zbl7GyXe_-?b+WFJJUJKAP)8w@oBTJ$Bh6^WUA#4vqS!=RFV{5v?k!v&Iv0|!~iLI#GSGpoDKx?!Dj_*wv7FX;6)&7V_%SmUx zP-heuJ)8f5u(7#2?*K%xO=5tWC_u0Qee~g!w1uh;=~Os~~jR^WNq!Zqr}vwRl28e7=D*tN_B#v+Ru zYi18=&C;gt9omB#$t7&D^kzP-qE7u(M+XQ)P0e}P$C9;)i^-Ty!Q~sBLYGlmnaxmK z>3cKt(%I|q+PMzYCt9BCTVy+1qrM5KW3m|mKeuE1T4WD6 zn6tY*JknXFh8%_*Awydi94nVhT8E-oGA-Ru&m2OJ;Ue)+Bh4< zSR3=LSkFE@7+Lzll)$5PIOSVl3`;7kP=KjxyR}XZlV;*-^j|^Km08qk0#~*6Ysw(t zLTr7oBp!Eq3*W$7dxX`6r%>!xqtO1N?f_p9 z|5qZH5t8cW47P2c%F@y`=I#@l!e*g{i%iUoZqOkW2jvW~Z$8hHnd8sHY@M4k;W#HI z6LZ=(8gcli^Ea^%`brZgIJB~#D=vpYwA>f_K6s^agcdQcuQlU8usmARGAd}Pge;5C zM>q#{&DS>Ho_9y;H-uFZ+B9;@ZchO`jVeFt!`{8!B1!;!$d4P#w>8#Cj!S+|nXpmDg2{ z35^%4Q6l5Q<%!CYmu73c-Me_v>176&`=<*Yb7m2)U&DH}cpxd+=ZKYiBww$;%nKj=>lQB6H5N3V!E2BrB(TSw(5e zOga3#F_KyRPHm+kiH2=r!LMG<&a`!!J@xa$vKD~$vGf0Vx);&)763k<#VWwdshgWq zFW|qA-~T+_nB*%v<+(Bjxxo(F_r5hHKSN$UPF87!Eez{*(7NMiD?53 z;T5=mqUaYE=xm;ZI?9wqo>78uBv?kacRml^ehx_*<>g!kaobMlwz=0THp|@2| zDZuH|60eBx!uM&ad$@7vo6pzz{eDFCn_J5@>gWS-YV>+qRuA|ot=`E6{A2^0BRsjL z%PXKOrvHrBXY_h}$)XDR_yVb-i2ff(ckz$1kQ(c`{*~qQk6BZtlZtgNe=$FfAt{(R zOQS%$-V{M!3Z37KwCC3K5%YjGBqzJgO5TE^Ch_)PCyIN@dol zW3yt|@FRO?8;ug-06V>MO!%S18zh{jhIAASRRu%lW+k|yHBWQGFXnut+gz6u<#mSu zg1Xqrf0MHump6E-42?B|&r8D*1(6tg1#yK(K`_@-DRgbfto9sy60s-?A}jGG z`t6Bn*aQ@kP?b@E(e-2(*=KF*g*&A{xlD+~h<(SWkmxUakpdwplXimVPvucnBHj;6 z(}AF1|D6aBBUH@$H{P*iTXOomV;8x)U7n&(^j@t+t7B&7UyM~`4sT1-gGmtsjY?{8t6$E;R}y@+UvH`(dKoL|%_Vyp|9Vb;_VC)xFLM$oSY zbGqZGsXlYW3Msd^rna%43d)0jWj=Kgz6wVG|Lo~tGSK%=D9pdS#J97H3FWEaITBZ3 zsq)U#jR#nEzX)VNQc$~U_q*TKu{_QZ{!;1iC}j*sk&>7`JZ1!o6M>LI;FOxydXI1% z1;MaZL`4eS-{vbNj2kO@z9L4%fvl5r957pNM__&8G3L7-5>j$Z%n1(rOsL5bEE&<{ z7QNxP$3e0tlYq(SfYH1*C@Ob@J>=!dF|26AB(ow(l88AOade_3=y%P+M|K&d_B{Ml zCIOs#h!OC=lhCXlZ}T5;-L)0*jIY!Q@#&rZr0GEXS%W|NpAj!dmV-JbNKv3%6>Dr< z5F~xt%wNj>L%$Bg?1wgZDKvg~$|hSxZV;D>?iSHF)+le0FQ1P9#;M=VV<)z4L%}LK zKTp80#Bt2enT5)m8#PC4{}dV zF6QqZ88ZkS%8uhyoC@N*8G+aa5Q1%l{CnpI$r5+y;o8L_3Y_O8cs&ZWlwJi zxGjLIY1#rJ;3B~sz$tmQa4b7D#)F4Q@HXW_012!@SiRG3WXuzK4`xCVP9)sz-d5n znlsfe$mL+Q{;;o|=Wi%6^(Vm&pqk@L9ZA9T90jHcsm)MUQbawiD_K_}3%8~_?7W;& z+4cVn@=BrjH`3@VVr!wWK%3(^aFgkb#`~MxDOLWDeVrm4gsKod)DA^Drbeh%!abLP zRFf471a_p_oTg@t1H=GRcN5;5)H?7cJiQ2Xi0nS*7AI!6m*quN>r?nZV{(}I5bm~5 zcn{F|qBpZECykN15AUn5w(O=#cb+8Z2bOH5A;~Rz1xv^}{b%P;sZ%HIq7d#+(P(%{ zSu4>MK)xmCG%Q63CXz0tp75@jE_K;C*E%e|qnr@scX~RPXqw7itTN>Yp7ae&!+5`2 z$#zQaaMqEEw8S@2AtWXLr9ijlQPwhIp)JT~@DSPE0qR}E7qejRD+}Iv+AAH1M27|) z_Q4YQ>E5P_K-P3dFSUd!^hU_>zobZSHEzz3(#~K zY~t?btAh0K4J$sfy5_?q_RvaWFAF&@)N)|=%D_b98cB?H?~=D^<7&tc8;G0Ek&;wZ z5V11M{@eVqo*U~vOhv4Yb^x}+=7RuE$NPcTo zQz^FPtz_jZ3pD7^MZ>kx(;u}%?H?DnIEMizAt?tIq!h_Dc_ja!uOKw#^y%g(zWE0b0{0vbbg*e6-ulTj>2RcObIXc+N(K~;=OK#_c z$v^!@{#740+hI2)_~>B`nq544S-e&LPrhQYNQEW^YFu z@i;e-stP=;7th0nC(t0i9>$Anl>k%lecoxlu1w6#DPGr%7g`kB9SyocWMJJ7DK%wW z8+Q9g>7#{0$^VD7EI{#-Nwno?C*VGSHo( zWys46oqJx5FG3~3a}ATDm!*VQA(0B2)M9NEWn|4EdLPw6 zC)o;_gf9lPGvc~3z%xqW!g0Lyp zWo4wyw_P10(Iqns+fcnASBJV~9@}pMmvj-j0xh2nIaf%Xn)-Dzs_2wsK#~=a5Z$y^ z5p?YlryaWp^t!au0vWdWkj!-5njyH ze3fC7n2!H6OUMti&e}0dXC**$JjGnog9D%0&O5+ch2N-;yYMqkIOCo`^fv;=Lu1NP z<@P_U%!+8`#D2Jr(!%k`^0QM1!?<`kjT?i*01eQDq>LiMHi0{+tDJwlVw*qm<3)KB zP5ine#g)W*viQCk-dJ5^hPY{*+R(R^c4oU)o$H+~hxQ$@l{Hwa$)^ytaU~z|0gMym zrHOutKk%y({qb=*DTl>1Ivkb2L9D(&qK^vmeWeXo^`$$jZE^&D#bQ0Hrx>4zhhi{H zKXCDLwh8Yz*^|?)lN_jVXR|QSmDHN-_(F4>ve>8tHra`2tR>Bn089B#AyP07XsNkl zo?s`wTG++}NdUD}M^H!Xallk?9##=Fy#q`STATLp7%<*jT?N|AMC`8g&+y3T2M*^M za6>VIi@-^J59ze`y>|vNqqz4V#F{?ODQS%SbEslC<4|`X(+oIAGcnBzNBc0{f8gsm zkom~TOy+75Fnc?#`d}CGNprGk4n`_rXamsk_{>bfECw(@;{obw+lud^A7cYwZyh(7@U(v}2-6T*=ov&nx?h26jPE@r+QK z7t(n4>3^;AYyr_UU~GiTL_1cnH9IhK=L@KsUV`?oTuGszP`U%utAFHiAXpIe!Oi9B zviKgu540FmM7mV^H@U*xt2rhAaet#HgsWxpw@o!m zUe*J)9GM`uHs+~)eXqec0_CYRx#ptK{`0hiUhr6b~|4eUHn_0bmNfmIfm>1<4U(br}iJ1eO29z)st2H3tX; z6?X7na7(f9kZ7p91)#JKniy&l%fyANPvvB1@8!}YrBMN|Oz<^obbgM{Rt-<{ZeCHw z22mk~&xb#Uo7?^6+U@CDW1#?|G)3{_oFA_@P8N51UFkm`*5&QY{1=ngReO!Obc`5k zcdOqMm(`dvO3&iuqAF(<&;rw>j81foYB6CKY>h5|dYqrHbw2E#v$|`G{MkKQX9a++ zJC}s?M!A+m^agY11McD*#kd)Fx}=e^7=#-tW!%I1kCNqM9dxFPscinQq}1=K9Q0#Y zdV1uuNJ&R1Os0>Dg8KiZq`0pC@08TRe<`VoBy?$c6Hyg`6k;YE93I{TzEL6NX5xg` zr`+lH+AY96p#44yB7J4NyZZHD_Nd4RAwT{qzSEgPcArz>Y+Bl9dx}u ztZH@)7vyUg&~M}XwlRHFYADwkGue|(Kp)|z^vu*Qg$EU-n{3>uB?k#(dLgE)r8dLh zHcP2t&5eto3*7%wP1huwDlbjuqt>b&gT}zE46CUK9o6KcAD8SIiL0k8VeZN)uOYTl z5WOW`Wj^IqO)-)Lua|RU%%^uxK}q3oqhTbSs-{UNh@3+oD%V3O!p=`uxeR2yh1B@?$XOkj-GhskR(0ia`&k zP!TKPyuiDGiZOP#1eP#S`CC8P<}h3@5JnpyK?&!1m^BwSF$RQ5?T;5g@>A{6d-|6@ zIx|T^?*C!$u7cuz^ghocf#B{C+}$m>y9I~fPH>mt?v1+>2rj`bxVyVUaChnLB>!{f zJoD5{?NsgV)!y_y6wtq--{03qiexcMy16_S?W@)5b@q&blMGz_Bzkg2rIxB*F zWXj=?Hn*pVaMzy-0oV9=oJZm~AF}uPVqYSSw4~7l>z(KQItedkxnz(r`3iK&GMJ7aPsJCXR00nizU77>Tw-}I z*H&GH?_02RDN_Ne(2**8j(!Dx@x%%vN~f@0?M|+cT<%V;h*>#BdH0;RJ}3vFKVW5r zZFI3vk{EO{i%1l9LvKPp*S%c8i?!z|zk7%0<3QAu{(I_`Er)fkvcT0fOSFef4P3>C5&J-zW>j1&)>mowoVR4QdaYlO6? zk|<%4HJhA7iKhW*@7=Bso-?gF7YFse_(lW*wD(@t8`f0i!(e$bUo!_Tq2@KCX?(-k zA$5=zH?d|H`gXC1-xNIzFVV*gtWwTf^~}#c-jXN89DUbz zqz`RNze1ArVYoT%L*vC4(gk(JAxKG~K$;&f8MU~ax^txxI8Ozxwg?6F(IV+_Ifq7> zmRAz$jA)3a7&yritdb--f*EIO)^A7!KE=}v5{0DuF2pJ^2#qzG@6{;c8cPe4#-V3S6TaFPjYKCl9`9s5PEp~B~2%)CV_QROBef| zRmk7LsJ2@%r&AK8sCi`E)2dW9-3`z&-R<4Cdu$ zx*{sl;I4#tRv*_8N=2W#P_uwU_xek-05uI}mkk@Rm7Nw0zAi=w(`;?GimJ)-;)XkL zuzjBgp&n-QOy7siyZNJUC2$Wdy-A(EVcoZW%&f;P$7BW3O9w$_X__BK>`o0^H9p1p z4tQ4TGEhXXP}$tI4Mog#w2r0E@zz%1WxqeZt?nA0L#bO^P7`g^P{ceix-Qq$Y^gEm zqp~ealSQK6&R2yHVtd(4tg{ zgf1pQeVKxqZ<&YEhYwb7t(p)YpBrQEi}eC;Wi}&yhqa$13aon%lsW6OI^fYqx91)e?iq zj4)DN_RDig7=8_sysP?HWphRLu7c}Ay0MnfnrxIRUMzfZZCBG988$^?Z&psnUYnyhVV-0zJ_+>bsea6(*jtudtQKZRd6Vb9FTE zljORFTT9)#5NwLmV&cu9lv-kRM$JS0v$5m(Dh=K0{pWE=9ZD`!Nzhx<8#X89iN`v_ z;3bnXn#9menTjc1of9W28J)HcD|~3&UG_4LnO=7vqx9EB!z{d?pBh;xVOVOp*r%o( z|F=mD56OQ|VqpJk5_42Ex@S~R3jo$orlOcP5&xHKCx<4tQ06H)5<(0hP4Ee$lH6qHroxSJ4Ckg-l`c`X zYe!<+&m;u@yMj^rcLifHOI9(hv-3{{BcFtkUM3qM*!I=SEIg{ZyP}$kYnnD`#@!MC zv?~Tyq@Y`TK&Oa)-KgP;@12ELzdp*?6ACy7c9*{ z+*e=RBC)!BI@j|f5wl~v(@EZk8jG`KZ`na5}QG3KM9$#g^wFl%)BK6FX~Q;UjQ zAELds2QSr8pPz~*JuMK|GyBWdk+p@XFkD~8hq)y_$a$DuB=pCItQPXjQm)lc=AWN8 zcs6{Nm__0Ew(bG{^L@;QHrKD7sxx%JM2=+E7#wfKW`!-vA-3A6^tIR0?29&DuQwas z!b#X%#*DX>uGxLqfxHi5!T~ap$ym$(CV}xPc<|?G`=$DJGQ~V9-YJ;VdCCB(heM_av&y%Ey>AG+TxgpS*5My{7*M1DLR9sn>qNSA_WQ^PR0mk(fCi z8l!JC^DbJqE<*fM<@2HgZTMB1jyAR~y8(_l_%D#Q%HBD}PHaO^T`wxrv&){bS~Bj! zk@3$K<4u?vLW}R{EFd^ZIVET#eHhL0Yd`fhF7q=*!t@g>*NN`cxdSLcXPC^F5c}yl z(_$sJ^*x$Em<9q?0t9GzzEi2s8=ly;H99}Pyo@|X zYD^Fj$mNqK79uj=hrd^5M~8r|4qpk0_a@b~$5kvrWRnoxrZ8D#>z}%?WtB6IFOy_B zXqocX=03t3!6`RrBVO;ciVO+#p#(nWIDY6T2DF{S8dzjx!^`_b!m<8vUHYlv7F$Zc zQi`-vq6I-w-@Qayi){&=Mpzne9S6TJ>_k3Qb9r@o_^7Yn&SA+pNKaJXL7XE{Z;ms% zZz&ZM8=KN_{_e9zz3BgW2xF%zhoRf3=G;mTlg=X!lEE{Ww)((gC_zLK<5rDCBK_XV~WNLhPnv)k!H2T>Uh%;vM4 z2YwW2`PURY9MD?)<}}9ioLI#UYJ&2Ezh~;cblpvK!G25Vn((Bq+re!r&vrSp1H@LKNXBl8Rsjj6c^0b6=KIR+Tb^3$c3ZzhZXlMR0g{I`B90CfpSyc znT1o*`EvGp4{zb?R*rmlbs8dH>TeZLr|cS@S%sCOxpZ=C>mQ8FcbCh>7l#mhlg*CW z24_eSr`^TYJANvWjAA-}gFbmm)ZsZ;o;l-uw50(`uak9ZlfZz!oDfA&kgou(7<0bY z+3IfV1}w-9sr-sqnDlL&7D@fb!F$Vbm9UFbuE!`D>SpLxEkMjAe05e#6d>!RxO7cg zz(;p22e3Ou8v4feAyA5^l(q;g5`IHHe2~134d317a!zbaT*rP6TB;1T7SiEkCuhE# zB(ih#^pw+fOCg|24rRhxVU@c1n}f+NWKz>9K)&;0&X092GI-i0fntFe?=kTP*D3cZ zyTgxew57rI0B54pf1C5P#n>_EK!o0HNEqlHgk8_%!mDTb~g+us$9ya z4xeS)%|X8GQ=Af%`vQP$9%e3;skL@eQ@ma zL78C2knc}y%Pu~x)iwAl`+7|K;1@1}O!mFbfu7&T_-DII38Aw;h~Klw4hm;}(=e4- zH9ZqfX$sLF!yE`bzVkEzW8EgSh(*qgF{|i+egl(Myp(K|y*gen6~LB%kf2f-s#Kfd z=J#Sy#J2qrMJZ-Rml#NhaVQS%GGKaj%zp*s*evBh4K10<8J;qGmi;F6W-mopy+Im7 zlVO^}5r(u_)dDM50WIyK{mU_pXRqm@JG|yD{YS?Z$t;};BtqeR-^b+}) zORHxR2T`)^-fD-6Rm<}MQZ~b*Z$Fz0^lCZ~SpnMS&DZf9*qm@6djXVxdGh)OF+Vi% zN<>X%yLJvLdtX>y>2J96l;u_%*(fRWKSR*V;f8jZ-QZdGG`5g{K8vr%RAbXt%YQoA zmQ&}OucxJm9J^vSpvPEUs$d(Z0KP3BcaFc^o6~1Y&XA8SsT-Fs17nFeu6>bWLIa;) zK{N1#JO$!ij^$rMBcmjO9dP@Uk~j*wFtb?CKGr%i4Ul?eUf@O=liqPHyX2dAN;zW9 zkhYkiklVBsOaD3n1j*MFo(V%sCu)Y(Oo2Fqno8&NpD7UJ@hsG%1KtC^9+zABQcn9P z0Bj}B;Fr@rDA~ws>0+YVOz{J%}4m~x2ARO+g@iiZ)7uLTf zM$`h9*`0A@yW9aE%VtspJ3uFr$;R{>KuM||Ny*z+LGP=FsnE#jW4Dq=r1|!czcc z-2+{RXpwIhII5W_>bEXH6;2P!NEz;9St6@Fs_lL;MB`zud|W;wblRMAxSvf6v26XI zI2h@K<{)C6P1|a&mZ_^lJl6Yr2EJdiDPhop8NEzp4*#G$OPN;Sigj)D(8Z;qGpbNlpA^wikFiDd_nkNN|v!>p|iZ6F)x@y1S-@hgLKl$L7(V z3sIV_MJWfljyz3npw;zEyIuzRcUkeQcll@X*vAG()ODePD?KZn~`npbsL|?t9Wy zr<55C(t~%7U{hxfN=3~ib`{j-)Zefa;U|gwF}tW!Xj++Mg`Y%Qf%6mf7%_tFDnvF< zskD-D2)xxS{AUS{fkHxj|7E~CmV$5;oEvhlfS7!U5%&Q(nhU|OadJF>Qoo01ADYx? zr7I6XaRHnFq9(Yy%88-Gd43rZZ3Y8&oyWb&WkGmZO0>+>Tc$@G>Njtw*8WW+;dfbs9>JZZ( zx`h@l0%GNZh$sW`uJpUXxTF9IiD2R<+-Rz~GmR^3+AvfY{$<}1p>8%k{2Y$R$@=l! z;M@dC^pA*p-yQH7CDf>1v*S_w<(OlX@I81nM8G()cL_jfgoBOckA((pr*`h;({lI$ z4Gn*ESJ9I?F;wo6ZSnRlap8oxeUbs=?O+023oONY8E=T12xDn6O_Djy%t)p9D9ixU zVe|@`tzi!sAtfKR$nd#-aIgZf=j;br54udnT{%P{0YFOBdyjY6<3m(eM_jxUMiK%% zIbl?Uq`>nzU;uQA_I49_4h+Fc%Y0e9)&K(9?+JL3E9u`8@ZkaoREZb}{!*vC`oUh7xkkjCmN5uzv=S>LaMBqOjishj2j!usb=TFp3Fy$FDXmIjG8 z5k=x?U2~kFAkzKkjb}cFe9bA6&PN$M zsufop;@A*eiR{dz+rB|`jv9PF@{b%6^gL(B`+jj{4c(EoPt+Nx@N*Z_NWGX;2MXZPIKRn9*Trvg?#mom)1(m-7RQwdgiD#1SbbmrTZ>Z{tiAj<{LetoA|;{ z;zp_d)!pwa7;kG(hHpG2-P)l2_q>C7Hfk)~vY(BCi18$aO!8ESk)A;k*pI>c2?mLA z<^W8YdYb?eU06^wg2RO~YIJnPjFzlfAdbZZ0>?z{-*k`6X8u2K!w=PbyODCwK9qh}lFSd9nzy;vikxm4XZW z@|L?HTvZ1rX#PDwFE4HKnzvBt``%LFzGsZGjTYLcBnUbFjPEF`N#KGqE!EHBeu+a8 ztp>Bwp!xSOwT3ALx#PZ@})z#h*KI!8A6YoCdr`rcYLd71{9KIHx`JRx;={yOyg*y_YhjXIV|=~IA5f- zgy$Q;ovUed5DhlfpVZJ3(4IoFIFI}2Gfxr4DFe*xJ?3G0(8Wz2HDC!Ql#=8 zCs4Lu99EB^urDf!&QC4%^gIp&J zbI5G3SetKEwUebzc@po)Pi4p**ker=#pMgxq{@wa8!E?i85dh08Z#>&P#ZV{blBeV zl4|F201i+skgj#iGe5z+OR=4w31p^$(SH$V6 zMkZAvG5#|@c$D(ikm-9*P0_lNC_amBFxw?(BladTXjh9_&fl ze;X|z-~Eb|bpy6E{k}E#T5|*KK;The41iN8U=(;#O?ZPg0NQ{5{(JvjIPD-Ha~U=H zIaAj-Wsnm$)B^lae19xW9W8*HyK+&a6@8x_{U8$uGyh#pYfrlJhkaNKs2SGcXmD1? zD-zis2W6w(veK!#Y8%=%*;zjtL=?Ka{h6(BM?`TNJZ<}`^ zciHi%fXm#uN<1N-tmK*_*L5ukCQ@1=&>h)r>Yeg zUFT+ur_%upjSU?bd#UOvki35e36uOEC_?BEB)y%8er5P>`acgG#@e-ko+;$|0yz7Bj334IqSqmLr8z|9bKOiqiYOvjsu{(}$W8``Hn z!4A1Vx@7pxO#X~7?d*g^#%VbJX9kkGGiAt*453ahtCw{@TN1xYDL}2oN+ixn8X;?h zKACL0WvqNH+AgcTbwMDL-~61;l6ZWb!5PKx+4dF%2lV$373Z?}hk^I*398~K7gNGJ zDg?lZ)h9wDdk@pug3J`B?-BSTG@4zYhkm?=z!*ZccV!2`kKH39Vo&+g1R^1$9iVa`=Y|o_E=MH){z2rtywB3@1bz2 zCFqbs)`D8Q{d~2kZ5=iH;JVk?(r0+?1?^$DW}Mi#BeG@SveC(}w2`Fc_ANqYw6PIt zxW*1sk6W|iJ_MHC;((Wbgl(MTx50BCI1KN(#wJI&?ONOJROB@|Y0p8mt=6yUF>uI2$h-&dmcGVM4<%+$g+5LPe*uv?x{_r^jaN<=k-5#9!kV#~>{?k! z%`Tf_li&?W-U}$GED+FkbSnXJAHFJti<-r@$!U$T6f^FKkTHKbK0YX|16rbErQClj zrjB`k*nvS7$36h4o-kYd(tf=HRW=>>svWT*UtDUC^%IF$PAevX5>fIvo#p#ZaBk@w z;QM&Xz{db%t#xgo__VoC^lX)x4qs4=A`MjFdI)*U*>@dTh5BW3Ly5zwzppYqp)2iB znxbK-gt3vnnvnZ*OfyBQi$6=}>Bh{4?GW(%25@Y`p9ugJ5G)$TlKmFq(k^>P*4ys` zg)|H-$_{5m*3=on@uF2R4Apjj65ADcf~%Iffv}s{h`g03aC%r7r{u`Bc(1PZKz4mq zD2*-V?-2dY^jJwrb1Q~8W}A37bdaBSFG>4R3t^=~)}k)CXt8Ll4h$<7?2`NiJ;vRW zjZDJ*Wb|;hJf#+ytmJT8ihz~dg$UJeV5m|tq}xmcp+T+*W*oZ{F$K56E7<8(E=ggEtEI13;NOJL_U#EVN^W7aO>*hXwloG6;4D{5J*EMtD!3u0= z`R3Pc^X6$0mRWD;f!)@u{0-)aAq~oZz1=MgX+T-j2obMbqj+S{!cBz20c0IaT8@M& zy;j1F#JSn-7-OTL7A}cgfWLV@4qi)H=+fV8TV&37Z$-;?-~^fr2Y{WW4IN`W;a zPwL87Z6?^H&Lt01k>`1QVb`y6GOX%xX})XzL}uW4@$v)!VL+Q zONmUm6sc+DOCN_Du{Xj>${4dkNqL4$j~)E!Ampo)*HD%yQ_|5m?VY!uKP2Y#MNI<6 z<-Rzu7&0`jJ~icwnV~+83%SBN>-YHPTr?|BoP}Kj{Oa;j>iDF;3QQSEdnfX{M}EQM zC*LxHNg^})an12pQk@yhwr-%8x8v+6@adO3DD}Jry*vEBUBaZ`ix2P9n486p_jG{G zh}mnfNF!Sf7C73348f=&(*wyKE8M#%(XlYdw%-y7Ua%AtDDUH05y^jllHU(Fhxqzoj*p+((4uE6BA`HL z+mT^2YC8Anf33P`sD;rZ6x-d0%HP}1iLTVmtVI@8zEqV@s2ycIsWkc_ zz!Db|GVKJ-4y~N_Z`-Axy(KKS?!~*{7xgf<2*yRY4nS0#=M*nng|z|k)?mSyKMrr@ zSm|H1OIfACQO62c!+;Bod%)-IVZQn^ZT&ngMVnJdu+i5G{}97*Zs0{1hO~8(^*6=28EcdupeIo|O-0C)K%iuLCE}wWLV3vJ#%it(; zG12GXL!Li23`{B`F>9@73$?AvrXBCw*Q%*@Htvd0TvcaFA<*ooM_S-h=UV~8VLHq= z`?2rUa*9O!gBLxd-}YQsmCHD4_LtYk(oHu z9`B;~C`*$C?yebG<5$;~`5w19lTw8_@A;D@PC=&yyPSy$IP-!EuQGr7F6ob1^8ARa zrH=;IjXI3Y92WC8)~+(;9G@kk$3kpt203+;T@~sPL)f}g*$+u&&w2~I`mh@Ps{u;U zO|;xr8d;@p>esovyt6NEHSb*S?1;`4+o6w>eqsX?>u*-_Z#1ZDTbqGZsBzu=gRMVK zjDAHtZ^Y*C***akVxT4O-#{N|`4QIJ7=@qr>KmOV6}_hDu7iz8yQFw(IcOcw%5V@csZYjXIq#wbEMzTkKB5N9<3AMYXvNl}?xRcRi=Egy(szUg$rkJ}PDn`W#7@#b zS!XBX{$9w?h+Mr{Gfxh@bMAN`K+9BcO> zg&MZNCIf*~zfRLV=y27!%f%P6H#;G|jsar65M4tfyI5`OwG3!X1u zMvAe&gO68WS$xX1Z7noX`LJTxKDOYGnnyS2jWR!i?qWXP&#hy$_d;tyUI>^9qR(mQ zBr#-f8l*f3XcoFv*>|kbP8tMiVvMpuU!H0aewJj(NZ(44U@u3i$2ix^X)od8^?I%K z1f&en0E6pvp=py@@Byz*mv`)1CtP*JKYTj(9Eus)+0`{-nzwiQvk-=kw7Jj)-hd}x zZJbuP<1Y?&s(vb`MJKV*425Tjaikr1$Cy*OrA3V>xZ2TiU<#9yJIDic18NIC92|9Q zbhhbVVYGmwkPEjj;+gNyFPAQ9cFPt_S=`{aa2@*;Fm|9FNOCxmE2VsdE+C06dNJJI z8-QX0|pj&)?%X!Pyp8+Bv zBM}|>Zj2lp|MCTmX~b3Ncyc75MuK0&v+M0-U@<)wcrFc&yEp+8ya7OIuq&A{IYU_Grd(fHl@6V=ayuY8($FC36)D zxI!%sTLZ8(=~zZG0<1%ntAgggG(l@-7|;k9b}NU>YoFgMk4Fkh_MN;xA;nk6Om$na zu0kA+HzIAT=F#b8IWib#3;b*)i07EFue5@&AQb7-xq`EFhoda2s^fV~2S&v0W4vL2 z(CMux*5QrYz)AP(M^K};aA7C4vBJ@->8H9z_OM4Y_DUX-x1AT$7+`cOnQFeKE+Lw?{o-8FVO9h6hkq*ARz8jyR zYJdF>Ybt{@?Tm zyO)^>;xw!J4GNvg!~M^!UK-`cwUuta!x~nRkrr$Hn%!Z(qd}-W9%h7ltqhx7ZTE zq56rqped>ilF{x1rub6Dhh_)wAh@g=RFjH{n=u5ABx;@T3t-3*hYIu+6ec!&pWen+ zW)#n3eED9dzre)45x!m#75i!}fqa=L_f;H_FZ1SlITL!FFryKBN2!b^GYy^U)f-5i z4(tDJvDZdK+7<>|w;$cACu%Cr9TwL)8YCpH$bHh;ebO4Ox2?~Hj#=JkrKH1Q3cJ4_ z{9o7VfYKjl1j0_gs!b7jHh1UGZfDuBFl{G;U!M z$?y35#D1Z5cBLw?rQPFWhb81r^qP1kQB|L3T|v=;_Nd)-53P~fM3&k~3I2pNCOXuD z4aOQ{WW8REcRR?IVTl9TGR(PS{&e9-F@J5DBYJD6UkWj9;rv;aqi`nCwCO^&MdGuZ zd#=VHTZU4?j^~Fwa8VOOqBB`sXvJSr*}s;pb9?irmBosB<=ush9);cl{w#scvDu=! zW0gB5*L&UOcVnkx2BlRyZ!2IV!xc0$z_`+u>hIPUTi3`J&4EB{+@0lL8}*l+iaGpV z>i0kaBEGK|!1aNH-?|LaA6=%()5Vi$t20{iU`CE)4Yd=Oyf?G=^u*3VW$=X%UGwCscP&t5$y$9WPs68{(FE zEWr$G@^v7)IgQxE-I2EhMdGU8TDkPV16_4iaulHA zrMP@e+UL>q)mB~|W>j(a64w=xZu;X~`4pw14J8p-9-H3hw09kdrLAK#^s|?n(6dS(in367Ac=n3s z@S~aH9zVZ7)$g$8n4qcy6xNJ^!kQ3`-(ig&;O(Pg!rh!R;z^q2^NqG%I18<#7`((h zdFN6z&Si%Cw>es=5{R~pK~(&pqXASV%_5au=*vg^NL)%Vc^*+UX-&Yc>Q1qB+(>`- zK+i#$Z$k>uk83)>qxNSj_U@Jy-6r0n9mxG@1AcwGApy>`d-Gn!r(>BgnZYha&fGWx zP~DQeCMhbnG@SZww_4U14+EnX!A~zyXQ#?%7k&~EAJV-2YLND6FC9I|huy#tqg_g} z!W%HR+db`$PM!)H{1WUq-A>w0oqN_K=d(rvR1GYf7Sqk|cc+!iIAuyBeVh^S=oOIs zjmxmmp`6xq9#U5SfFs|t$r~}v&g|^}Ph7?W26R=Oct-`oWqAK~HCCXmror}f`FB?n zI>GOK_7ljK{r%&n!b_*`tlbr`;5~4n-P;H8tutD`)0ozv;$;=+@3^i0`qmyYKr}Pa z9R{+=dcO)9t6a@d{OnG2Nc|#FRn6&bjN|BiC!wRvQJ_XqI>9(@(r5g+Or>^_UJjAKhSdR`|u+h=wv@B=*ZqwyP;Ik!aP{qP5uSsVEWm&s}UgUkG)5hBG6?DigY zds)EFVT@omR|w4%TEe(0PJl;~;BkAgRK3~kgz4Cc1!muGb{<78{h)d)c-z&Gt|G9I zgY3E_CSLK_zCIrH*p59cy6$D#-`=n!l+U69?l}+USP&M$E3CH(?8W5l9qMZ-asOTk z>Dl*q(irI67RIYx*AQIA_vYjW9uyB$`SRT!Y%%O>etkoZfmTGeI_L!69QM5WGUX@~ zfU+8bGEi3I^CzpR5-c?u6g=-j4>qjgijG(^LO9AOB`L>qsk!4YhoXus*Z!Yz8TLQ8 zOcDr}X$Rpl&=e{P0`u4nXn%1TyFa*$CkU505_ReNgUe8Usa^)*G8}(#8UB5807X|H z=@b*9i*%$GB(eYxB@u&FmJ4nP|2^y4>L+obRt_L656{1FndJY1%W(a{W#~-*2QK4{ zl1`fkOPeG8vpOL;W0N)nEl*WGigJ zRx(T*)W(vK0oK<)Bi>aIM>Dx)I8Jz7?DME#+Z#bla9BeoY{mvU@-@sOtv$1H_hUEw z;wM3U3W0IQBfEu@B!qyJ{rt7_92rkZm$4t=bqX(;qmK5)xF!KgBi4<_uUo`Htv1&W zJ~?VOSxw5{tR_~q_SaPUUIcmqBiv3GW=H0p3DZ>? z(@0B#GTY-w$uajiVmIb{<*TgBA+`_6^s!doxKIbwpZ7710(mMxVuP*5k!&NK?)4cVRAk@a_(Z7M8Ls_C# zmv8c8p3jX;Pk!=sSGhdOpJq3sC*jEeX)0o9sMIW*+yY)ZAIDj!R7;d2n--}Qv!y63 zYpg64vxQJqgr3RGy}ym=C`WIQQQm~FQo)Zq{)nrYz=ek^Opn@2V9y~#zRLX;ACx;D z=?4|7tu}|6`W`AcpxKsObzX`ZKrjW^JCTtJxm4kzzk?XUh0Dbviu0&Qo2N4<$3Zli-q4MhP#Q^VnkTr9vd&EQInv1QcTjEX?CF) zN!a^yzu3=tel1424Wge3?q6s9y!$_FnT_AJ3=XWR3L*!o*GU8gM1fHNk31H8ObU4z zu6)qXT_%Bip**6|tHWFaF$GqWS5i!1TpwhR`J>Qpex#qhiJ zcb{My5L0fm-*E$WxtFUL40AxXOzVHzGDfJnR;}Pd4T`KmJW}XLbA#q`PWTS=m$$z- z?74ywa2jik(wJpd8@1A5t{~^DBa$OAD#o9~Fg?JwHK${O+n7bkYkNyj!~_d23;AAU z-bA2SAo3)ikujy*2dsDM%urkAaetsVtC$-NwHn990p3hfg^PRzLztO> z@A#B9UY82MKL~N(7TpDU&2{E>Ny!qBIf`yB&)4fEjq;ksZGsi)V8!Fp2>6pz^)qL# zT4B&9WF;CA51-(84b!1jZAvd-WL>Jj2rHYB%c}U&IfZ}adYMl>Y*mRzMft$@_02Iz z)@t@&O$}4WmEU?C;5df7)Hhx&R6YNxezkT?$tHFVuZw&8BC_tcEW-toW$b>-GVKb1 zP<-@gDEAgvkJr_oqUk`g4Ba1DX63IeW1lShjgs-3iW|wmZ&{{=0){*4k1X@$w=Bc= zZ&}7O?Vfp*vu6TjUs4R19vHpoCOuq=9%S zyEKZJ&$x?sNX>4$f>XT5nY4cW_keVglg(z}q5b(_yemyi>FU5%4vtNML_g zm5ef2BU7L@_Nvm@gE6Q8ia8*Yf&?zB($9e1l{-Dea(GrqwJ!1YCZ|xyqd0ykEMveGqGhd%LcMp%0xM zS3X}zIh;w1E)y&DjT9uyq$U25Wyb%TEaM82Ww=akt`4V0o_I|d#cHrVpz8n}RN)y; z3emrc{$iG8ql@eq=otz0ZBxp3i-Clh&PIfAvfp<%BBb5QrG1Y^J!w%e0Id7cT8rG zAa|A7dImGQdvZij zQdIGo>m=m9GmRL0PlwQui88&=kfh7OnU_fqgcMkbx=NBJET_iU-$fVb-9VqQgQn<=Dwzq+Wkz8SFo@OaPij z^6(nWx@-gaA6dpySl`yID7SI{u8l zBN|a5EW)^@$-PY9Ey18aV1&bjab_~_Ny{^?P)tAKBe?uKQNI$6A&pIis3@HogI$AD z(gK+VOBRcN%Q8j($TFt?l4UZ76Wy#dCy@qZKe^x5`@`X2N5PE+WiOBh%P>q(a%eb3 zS&hVNmSPx`F3AONRw6toPlv4OL;aRz5dV>7%It|ogjDmy^8~3+guT?;P{yv-n(d`zU+hcLH92tQI=}Liwxw#v zFmgfT!WP{!N-&mi5KDiT#tXp5sG7Xv;M)(&@CK}U&>+ed-Vw-n#k*3(bEiwPp5&8G zGkOpZHLOZeCe5*QUvIw;Mz|JC_RL#TO8 z57g9De4*U4Tp4sEHfiQaaI)z|Y;a^sPc$-V6#L!O2q^z)YIZc$bmIMOYASv=HC?12T84v+p2*2DSd?_dgdiWP z5Ib!_Lt6%Bd%MX*_EaHtMsC%~NIeX#ck7N$!bOux34?w_mlJw$urHK;D!WP?E7^f{ zcnpcZ%Om$QG~QDtYVuFhp-oJmCNO^?gbyVBTxNd=V)Z?^wdn?ZnKV8>7$xiFp5NYS2%P|28#&znhwzhu=*N9hu(9 zpQa``(8Yo7Uro*Q@1~|Rbn`xR;^+Z5LkiHNFEqMX@!Q?$EAS&#-Ir@Lr!ilNz42@W;e+l8Zib# z(E8ZbM2Fx-jz{f?KYDT=m*`Dyw@zsF)qLLervCG?upj;T`+=qr@IJVn*blJ0+3Llb zW?#ZmP__9#w6lB#{v?`Rx_Y{{TWx79teHFTwdZzMp8zfsArKyQZj2Nc6!hBXY(-S% zK)zlf^V$3Iyj@msNni6Y1m#i$;Z1FFz#EaseV*bI8TdMb28|m3MALlEy>}C5pnM1~ z@(Zsx_%c&WoYYD@)G;7Av|H6^_+end(tgMhJaeXIXr6B8k|jBu1utd^^i3-;4=hMw zdjFnDpYSHoK!m19@%g6D(Ei4|TMxl=PdBR(^89pj(gE+k+cL@jVap(eAJ5mhEj%P$eK^(>P!Vo|hpc+@hmM>FD zTGPYAP?;aJm#|EZetF60vVOSYHN_mr^!!W9-7=%QOZ*tT~s$&BLC6btmreS*kewu*FH>)q^Y5CQ3W{;@M8doqSsHjyz3^ zzmIcg9CrblqydJqh}n6Nr+_3S5}Z{Xg8!YgnDn8~)MpSaLm!_TCz6SJNJR!)4)D?X z!MLmlx99jxo8Vy5618G>A-QRSq*;x4-}7V;r-GU$NB67zivh|nJcKnXo+PEldL6%S zWSHWbSUk*wDhFqp%KB`F+BesP)0svB%Iwn;!Hs#bB8P{M z#+VV<>=FBu{28x2QLka9-;wuN_CMfbMq+SWGiw|-4*$&IGv||KsTC8VAd0f*r^v+S zC`MSD_Dyk6f!6E{gdvwiy{!5Z)Od32f_@l4L5=S^|L>rtWAEQVO~OAxO%*{ZdGV7P z$d*|yllW`PsEQz$f^3%A=_4QR`xFR=VqTc(gh_iPu7 z7ZwG~ehLn+vWC)ySPG_=Md&C)ogi3xl%JUd@F|dFR`8A}O-et}G}cI#J0daJ(L*f| zgkz$T6ZSG~DQIsu0=P$!x7R$qG4>uIo_XyH3=OK<&K1PWx; zVJx>rffbhP1~qWDguErHm(j;>eF953<}C!L@;A|CL3-uSO*rBaw;A_C>AsgD7h;li zAj#l5el>@Ckh02SO!7X9NMp+8F8z3D z*f1i8^+IZ;@;glpvY-LJ=bwH4*m&LvW)i3Qsput^s)hYhKoZOHXl!~(z`&movPzj3+&-r)ZCXH+n zd$uu8)fH0Lrkl@v6bGsW&^86rZN{#WJ+kh;La$8E__@i(P#NkhcaC;Cmp$kdxueZ> z04Vq%`lMLpzOnh^lW2`x4eoqZ6fS$X*7<8`I zymZC7=1H_=A|+$81BrDsZfTtOK|}MkLyfG`nH~t^>wRAw1nGp8DKsDOSb|tn+=Yoh zY$+!09^66=mu$0DI~)#vZDjA=>P?g7>-;3{bHU{y?uk#omp3|GH?)qGL77zSjBTZ4 zJ{lG)rFA)pieZ@p!4+zgVv0s?h4;0^{T=R!#e6dEDrpTfaDYPGr`MI$6abcFxuDPZ zvHK^|CzbQRHZ`{ZEGgs1I3@r~Y7)(CX$nFIq-FL*L4T`et--UuZE6s$RmOX5{VK$4 z*0i?ERIBE^z0|{<*aJlXOj01pI^=_9nVY+O-*iwv8@RY#=y;?riN!+Dd!1>7P66gp3V1y?*j6uJ-Rj$xeNGq)cieKl|OZyNIlI!>npy zl4|o^XV~1!kPsifa39VbE{{35n(;)kk>f=rFY(l1K4`Lrp15*whTHhP`cSI6;;{{p@f}Wx;o;|7>c|-!?UF zu5F9suB(P)f(!2Kc*{-vm-FlV+1comnp(G?P&NGGKewe>;w9-vvSZc|b1u$96iYBduiQmT2oq`z9<>EM~*w0r%sBBm63#vGm zT%Q`k9v`psufHDfeXjuPx+|_ITt_SH7K&Z9dGcwLM>`Q5D!fJ%Ug~(|_w-i3eZ{|t zhCqDje!P1x%fJB2GJS7lnd&cZWtj`Q>oP!D#v~y?_&;TtX(F&}9y{yzvdna$&ike& zh*W|26Nel5gK%Hrre(5OWkDstaeQWy!^Hj&7aa{K%WP+tZxU?1lS*}BsH|;<_bG@( zDL32O;AV+g8xK9V{95XKM|D85eQX$6S}D@{yOL=ol*x$uc7xnHiDnCI7`HQ-b$!pR z6K?wBJ4MIlBRm|Z21b2Z`>S8^Ug@OzC{w*v%4Cc_Pm&v&idxaAZX(U{Drxn9!#t~z zR{VgVUI=kEZxabE6mbmi;>?Q2-t>DeT^V%$6K(^Yyz8oBZAsF0&GM#IZ^$&du`1Nj zc4!%knN>z*jiWa@D~jT{xsWQu&-FM)OKszHkKUCWODd;UjYz4TZU&e=Z2!9(+UTmh z6xoS0X%ZaOV9l`8wTw(<+{{il9q(9%IHQ-ZZ=)J|Lw3^Wsln#q(F+GYSfTA(9$nW% zL17!$-3%-4r8Q#-)uk_~XXB=;cJqU3TTdloQDBFT!}_cRwFv?;9ZxMA;2V2P-mpw< zhmz6adkV~?Ue5_;p9By`XErzIgx}IKJf`CPZ)q8_^9w*)h6<3DNqhvPWp?!m+#(n< zJ~w4MGVorcj8@D|Ppo%LBH7%)GT(y(bL>{ZU5zyz6Yk6{8}vVYw4=n%)c1% z`t@}8(&b7Z%|v@-k*+gsG-MFyizH6~DsR)2X%_Bhnqb%JtN%Qp)q4V#?42gwKRU+RD={2Kh zPzOx=F=lFNd@k)cv0UJAVHhu9Yxo+ZV=4R1So$0bUk>xF{deOMgoz{*vhMc` zx-Bdh#@*Cj`o<6Nt&&b9HbGW7qTTB@5!N0NB}8R(o!98ThZ)q9c_q5Cl26;8X@6j}u)iFDyk}bN6yM_GD z`pfK>ONMOzjPBp6L(TFYm4LKNC)S^|%wUT^X7_0oh{n|$uB6siUi7PU4}dEvkD(?{ zc@E1!Hz$dA%WtwWX#ezOtI$-nDdH~m@7{Q`mC6^}X5Vb3v43o(ae%FqGf~;AWmd#Q z0TCZg!5XI4JmTzNg=F;*5JWzdt$VGGe8m|pxFLSX@p51mBh&cfG>&e``I0`hlLnBM zS>nmbGFQdUHCb&0q-9J90BMzwYy3{Ua zTu7`Y904_J4b8d3lSo(=3cnbR(s*-Z3=@5pI_e9u9gXXcqRuEC>d|XX@GZ~S5}+&L zvT2o#DaB~Rd~hLIZjDp`iwoMBVW249KNawL-TQq2;BN%7ye-%R#C@W@@1y6sE3AW_ zP{ep`BP<%|ecJC1`tL^%0DGo*N@y|*&~JQOf@l)~A3?0Q*mFra0$}O4p&Jc`%q2Pk z;M#_Gan@{#O6eS*Y4}MHv5Pf1E&-fyDJmi3ce_m;u zUvN>6^l%%)>-{DvtFu0Wu+B3DPnd@>^B?&%3&+ft00?bhe#?K5G1T84za8H9;q|hc zwT{u?W^<28Pv3)i2bSv1rc|5kz~93%R&T*tmuox|Ttf~#vC`0EW{&xcNkC!?1P zJC|o>t?2eR4h+z1%l0R!KhH<*H-?4!9kGYmCJ$;>t9Py0zI0-Z5F@H!P4D$2inxe> zH`ZmqHFrsBjOIY3ck&rboH;}G#;o4wu%1w zi4kk1N_BOM)1rDkv_5zgVXlGNXv+SB-;{1Hwd(|<*0iwn_crX|GkXZVda>tIRZ0^^ z!<_bkOI#bXe-)#Phet?etW`aV0R_vGMq243v0ON-#5R$ek6*4(3e7JW+3$;qfV9le zxGl&1n$$~thWL#J4OCV9E5y{%sQ>~Z;)l0?toChB1XrdFPcR|K^o+Gv9b6 z+?sT@11ul#rnxw#xU;$oIqN49$;OV%Caoln$d`Hop zW!_#MRf{EP!2;N_UrVL{u0m11c$?V48M=dx;D@7WvGLvUgA-nX;G_yYY7PDPx zRqwo#?l=vTm}r!F??~DoUI}%oLGztgLgg~_|HCUOhpiCu`xw3PN>p#W63rX0 z$>9+#<$v=^RRCV;v*ZWk0R;2tXh2z}@Sn1b?8TAFds$|P5Kxv;o&%g>li2-JmWe+> z|GO-Ma9@n0ok+fg!ZxJD^~2g{6m2y7W|ZWHYb&KrBTD0|dGxa^$Jas?LMl609s0|i zC$=PsfUU^Uc?z=_)Xb55iOI3N27%JNhG~m$S!UsM-RjF*StjDWEOYyxvdpOI z6^vCsM-UNU#MAKqUY22=LJ_K4c^O z-Ie(X=*pZEhviaRiTO)&o2gMb=|W;84yx@oMbr5GxWMB1l~(1%LNs2E|K?F1Os1A^l6{-pst) zg&yENJakW?ZDVKRNa@{HO8jFh@mXgT`?x&OyxB@#*ZrNnX3zHm0({=j+ob}Cc6`H! zmp=BaU)<&SUj6qW{evH0b@?;J9PEao)?+URlLphdV+|dwi25mi7x5&C|mTGJ zz`(6CV?kg{TI^B|*?f3;D~16?tN0)oMFrFm6%lBa87h$?+F0pt>IfKc8j`M`7;5v# zw)KA#lp4?EvyG!ukn5lg`O|m;vNF1VvNHLm7s(1s4r{mpGzfV+JY4Z++_V}53o>@Z zQI({b(L;jqWED09BdfzpFB$|5Rnv|EkJ(-VC{B7`;_xFdS;-vUp>qW(^l+vv|J} z7x>;tPTt9+H|ODWi_82%ZI#2w*fl3I%4WeplVHOhr?B7`C)g2`C4&~s!~}*VXe=uu z`Ku};QBn~rPt27^GQOM`1GimhtG)xvev$e5eERCpMIx8gAq~s!c`g3xx)8@w^Rm{S_9IUa-LA6to` zj^@8?C9J<}rR4wEO5yrJBbl_0C3&z$8;Q^z&_zhrxb#PS(DuZm4h?!*ef$QZ8?Iho zsUm5p_#K!{z&8hpTjoE5`dVijLpjBe{puLRqyOHiz z+b-%e&r#p`uF^(;4(cPd0@!5T7Y!SGg=x|Pw~_NS&R7=EiLFXiK(c)JkM#3F0WM%R;b&%*#o^v2(?e zX~smtF$sS@5IpjFTjm1@7$I$_%UIb=_Wyq zRJS+2ii&%?5Xmp^n?zEmxrfasO0$)w<_OR0bepPUC&+M zP9l;ed`ab3(sG1Kk!!0~ll0sl1m$dg4g;H*W-gEq5_WjS+iMvxpJzNWAz;6#lMV&a zZf-@5XGA2Ngz(5$@3NBA-?GxU_nWLlGW?}#P7?cn$V#H`vJ#&Pw&0f`KbN@jwE-;~ zfUNXsU|5flTP$yDd}_huXoT2>+u1qvhi7k#;E;4ZIM^eWwAZ)MQZX5rYECcX54$?U z7urzbu>Uk=PTra_VsyDGB=Wi%g60|IO7g!pOhJ1s^(3@_Q9%p`;N8$HkhqDtMzx^M z891bIfj_;=O5YY(=qLbNlyS_^dInpn%^ZU9)hL~l1x=GyrRAC ziPz_Pt$^wmxpX1)y6&E+<#%NvM2{G0H#c#!n3bzx zg~kny7U4sTNdDPxmUkCuur`6nrJ^Y5e#`dd!)6t^+^}M49)vB3c-h zwqSzA+Go`pJ9dj1Q2ezC&6P-XI$r~jm6HFKm9XArCDFfRrM)* zCWd+095;fy^gs=)_8a?;Fv*AX9uu~&ML{bhd1MzO{M3{&(QH_udi;^_Xo>{_ z0n?w3ie_Yp-4}!1q#PJ-MaoBIxFDd*lp@|UHG;@i?{~H_QO^`<_#6f$7hI!Ufe`MP zxDGo9&k6~{v|CuLJ+&13tZBL?u}>|B7TSkguKr;-z?boSALk;?rC)|74w(PUwjZI9 zL-d2Mpkm%NF|-DOf8r1;US^@egne`iP_EM_RBq8Gn@=+~KkcQLnQf%?z74}^XX|I9 zBVjX6QLa|VWR82{)=Nd4|JjrgeQU}X`!wYOnlhYicg@;>rc4x|DZ}*ElwryO zG-ZMSO&Qj=ri>*Tprsy%w=nhZrc7gI(oIL$TT>?S@Ox1HTT_N!IB>&HasN+KX2xfx z+SjqY6F@6zEz69}HO20J9C5TF-~SPUYrU(>N;NoQ8~R2odF8#+O2YtJNg6;amHnGm z`U0Sp2z2jj%Kt+vdEh}DfV|O4lKZvZ|Du(=+5XTMFE zZR3P=e0RUgkL_@*S6n2tU&rHmX`bwXLB3*f7hz+0?$$A48o289ML#zV(~WOI!4p~# z)U70oY;nr1i7z7|)FFKjP!IR4r)u%05h+iXZt4*HhgPEazVwGyf_$fy&fjSzv5us7 zT8R$wXiN2PTB+cTR`NmYJw?o3IDfT^vzX-c;fRcz>O_V5vCMgpf39)+vVVUm`1^Tk zW4TxOCEMfvV;OSl?@1!k9Pr4w(lDIx*)_QDM=>oiG6O%r`fRF{4WozGJBb0$mLsPg zUKe!OcXE(h+0m6cO8lfTYEySAaRql8XToP%2r9mDotd#Opu_CHmiw*kZ=z;JejA)b z4Jn0a_MxPi>}GrUYzSN$CxcSZ3fl$!5)5ocFxagW9_;PDY4iuuSVN3*NCpi<5Bk7^ za&>6o0{Gnu`a+d^JvW0pe~b9^)o?03+h_Y`HEu{X`?Ys-Lj&M{Z~QvTeR%}j7vYwI zyTz=!d*GU8cn?hcq@&j~?k3Y8$k>(&!u^v$DBkT|b!&>0{Rm3~Ngj8gi9 z2#t~0MHo=!poujlsZ5%u>P$&t?nWs?RutjmnF*0hi`m1456H?e|C5yw;O2H`=XAe! zv_HGp6%>FXPf&blkkiA15TBwr9?VgjTyWz#TSGv=FGkJ`-E4^)#;E<5}jaa zoLZHJw4;q8yw4oI8`EOTX9XY-=30Gz_aAU(z5z>_JiiEmZl4c{7Xx(oALWz{2aI(bw}4{c=-lmWV1{^+a}} zV!u}ZeL20frwU{_n0$%pFLNa}dss?YpNnx;!myFa#!ToBCIM9$8m2178$eYi$cp~G zDg#J0E3X)LN1z8Zo3R!?GpFrF@%#D{i#KGPFyZV_LG~ zA>x5hdt?GrcSW%+7yrS&(?qAURpzUea=ZipmdHmQxBd%QO8WyWQTk9-z5`2u39dBO z`9uIM3U&;#GObL*4z8?batxxbqd>xN?(W5H@M@Ph=D zrn2k;z2cs_91dQr$7u+*D_YqH-+?^aQxseh;|*9k!;=gxsb{}2h}&CYMHJMegkMAl zy-R1A(Gf4-s;T3-6P?zsAgmChO{@46yuxiy@(bvvVpn-+2o|t^b)t=6QB+s)0NET- zwUVmCx6kclxP<@HvTiPHi%tQwAr zmo){tH;j(e_k^Tm1~DFDjo_uX&ub0=tG^ za9>B;0+TF>%~)hqTT)T62G`FBx}>j-{foF7lPQ`|_9JtKX+*a|t%L##qEn-HIo#8z z(@;Gci8PqnPR$R>xBIIk zv0&7PTJ&Hf<&r@kYlYbE5sc#S{2iXhzDhcl>9KJ`S9e;XVVb`n%*v5nrX=tJ$cpf!kFjQu(<{n+oPc3-E{pZ;mP`h0tZ}smVRmq}`}I!EPJ3j#3xqrh{|G3|Vb?|bOIV@@ z2uo=?wVtnPA@0JN7+1VFx-+CmaOirUS?(DToU*?AW#TBfEA+wkHV|Qk4CPceki!@q zKu(otu)sKmGk4%xm4n45GC5I0$(1~fqS1B2iDNYL%0TqRw!AWu#p?@-Nu|s((i*pE zdtx)7w~yx^RMf~&s4BEAS3o$p@#oyfT;W4rHM3c^OO*y3dXP+DTGFS2KXe6cV!&~NlBTG5=c z(4Y!_=~dapmiwUF6)pIji|O>&&)hpx3MfTO8QS`mnzbowsN!DbbP;|gbBwf+MH`6d z$w*+iIT3ZIA>>9F#Y-1fEWXM`=b=iuk&t6oqyE^in6-}L1sSO%+;Z+9GN`?P2{vB| zQS?|NHaUqhU(FMuiQYMQx{SI!T<9ZxXADqCb)t%)#L8SZo%yk9?E;9HhR57@0I`%+ za49pgf6@%6k73Z0xh-S!30CMZoEUHOgN?~WNc?&_}17mk&EwcfMz=|+sAGBoVpi)pM^czt_oDWx31!dnve) z(@aU?%2!93^TI2qx-hL6k_^sDEuH>*34ryx?i&ucrG)2(6S zZ()gs=3Q9oG+O1;j+(wYZiqItumV}P)w0ERFkvuCR83kAT6Kw7q=1;6k$0c;9U`Q5 zx`WxyQVwUn8&wy>MQ%vSu13cfy%!1r;dhXBq@*g==vR^&YAPl8?Or9WXykz^_062i z%cvt&UjWC6~>kEzE`8@>FAcVS&SA-b8Hka;;Q%`>}=z z;bYaNSIsh%*0DA?Q}a(Ar^CII(k*T4EOhB{MvRKP0UK7Q(X7I1m$rw-t!46_FOryS zpeKtAa%>4*^AQZU#)1;PQG^e(P4ym1qczH~xAAF;aS)$YD$VIslY&WN_z3Ts3>KB; ztNe5Z@chG71#Q_K(_M?LIz5-@itv;x&H7_)j#%WLe_!5fY*sY7oU+D}aat6Ymll21 zY9_G$-84?-3Nex9J@F|tv8cSHc5Rw2X*ucV?6=aUL1!_ncyb-%(aJB*49gevNQ)?C z$ZFu?zb)n2*4zn{MT0e8>6>~cUw&UcXRrBB{3R^Oz6(p8k+0Plae%f=$NUSU^Tgw6 z|9-H`>-x-0|KsW7{r_FT{AUHj{(o06|6f-yI=_6596>6_U*|qtSv{<8&RhmRyxv~i zb3c5TV5T=dO46M$9xw#zAdOtQD;ku3DbZz55jO}o?V@NS&0wA;mkqZHc#&K^OfJPfA-8ey(JSF{Yf6FX5x9LMNCkXv_2!3_{~NHx zU_FW95W$^t@l8MZ#~)ycy`OD5Z~c^Atcec*EG;ChJ;1yHOW!%L`WeoO_|&>GLzE3r?*tg`9BQKc(qSIV64)wAClJRDZB(y@T7-EcDTXpjFH zAl-O&XOiHiHodpFQOPowg6cmy2)M;xu4UZT!WP2jB_xcxEm!Sc0p*@CO!$lB8u#^z zMWRUax9ez9*mwu&Qsc{$edi0QhMA8NF%%+cJAf^L)WFTmB5T~l+dsXKo9 z-c%NrHlBZgCGgvs3j-6xXYC30ot!W`HgR2aM}ihd2PR zq@D5w*lD7j71`2jmw8dZk1iPQ;n^Esj%W73KqJTPl@bBQ?xbU@Jue?qtQD9zp}3E) zQGS~)N=rjv-Q23{-saZqS5~1p+pp@CBOQ`D#n^HpF~E~77~Vyu6w<-~#F1#I=A#R9 zp!nT4Nt@C6%kLKNei}Z|Q+@N5NC-HFST&OYzS7m^)6NxO($A|#_lzna-pz=+pfme@rng>SBnJ3Z??9=fqUeqB)Y4YGZeI#yb#c7$N|`u(BNMVtfxXsQK|9O!W;8#W{i@1;5bdxv zfNnhS@T^?ZU*8<%*xIc{+5PDGV&savKWtUIc5Hsoh;f;IRTHMZ;ADo*b2xOVIVjTW zf-BlQZdj>Ct7pgYstacv8%zyn?zAD5*G6K^T%8Qxz&-v0uPuIC$574;ryfKjKa4y^ z%L2JkwskcH13f=dHVB)f^yFBw`xw#8k`?ZiRFA9acZGU*AQ&O zi*49Vi7Pa^gA>4N^ny+vEGdUaEwAh!LmaKft^^9o=&cWNJnx=BJx_LCMHgGhA6#vo zPUirbn}uzkY<_&+JD<~SC!W`8w_pPd=IK4PPVUa=v-Kg88yEh8L!SF}pR?*+^w%XA z_tksdr$eG@_z+j*b4$T~JHm|xkcFyz@FnYgJE|`iM@B~5(~_6t{{TzT@4%A9JFqlv z?+O5xpaxzE>&{nqyRXrKmqvZ2vX_ot@%cUeH}1g1y^=!4d1-<7x?HSpxII4)Ulu>_ zBVMd~d7j+7yqOQSW0^bmWbbgr6t1T$iV2}DYe%Xu@}%23D~_+lvCZ9;PMJz ztDSDa*An_yd_F$UkymGi?5K`{BJkyK4WXMY)=q8X0i9g}=TNJ=1#Jhf7Tad&2SSq1 zA3Z$5Tv%zY+0Gy(+`|OgY~u8)sV`g2`#2zCr9Ky?4V^FJy0zQqo3IiwYvpD*S4%L! zl$#2yENqmqA}5zHW&I5-HOLXvP9YB`OgMgn5%D;-YlEh(1d(*p0su=C`_;7~ z#AtRhMf_a;8NWmb1RYf|3ZEs>3j;qHu}eCwR%x0{1XpUcH&rL30Y)&0**xd35oU)7 zEugQbAFgqqUNv5OU!54T_55G$wwF#*VAN>@ZhdkxT2#7ysyEMvuj(1M0>qdc)~8Yu zJYAq`7)|iV(`RTZ;McK&@UlB+RFx;pRYDKz!&x-W)tAZ=XxYc-d~4wJ)t(-$ zPG9%-*RyNC5Xi=}nrTheo;-cvuiSsd!vjPE{#}CU{6DD#Bk-RR%%W2T|0|@pe=%g^ z<+_jeo&77|g5(_K$#2f#GV+qPb;@DOm&2pzGOUX!k24X2??#3Gva_ft6sYC|mQ$TF9Jk8I)3y^nLZ2oj2cRR}%8s*492{-FvZDG!5lLcy*pwyC@0B^0 z(y+@oGF)y0X*%Wn5Kts#?l)$J!gve-mRkM*OL?cHk=s`|(gpP*HCp`6ycfyNg8xsz z5;*`^a#lE_G=_2hl2SkXRI1k%k6TL&CGLJK5wjezMnBQLqI5tg`0yc)eaQs zHL1ucMuz0?zwf<%`sfdVmK@F`?9&irGEo8Zs(x(TyRiKZENQ<1OMYY|*F_c$z?Q!Q z?+k2a!L$Swu{fKA|CJ)F|8;tQPR!I_R-5X3>w+0Hr`@Yx* zGMI5KhkNdg*PR&8%e*ija!-2r+YbsRcv6^M$OMy(=cgUW4%8+VT+VQs*MWJ~9#`&! z(RetACXjg!A2^Q$*AaVLZoOJ=e877N-_}NBWf+)Jn7wn$_T-fo&Qr#5B+tf7_^84% z9$?YqA(#e6OY=o;(Bry}^=ke@mU2g132P*_w8Z4z0Xl4~2$&QcXVlc%U6j)|XmrbY zV)~oWcnU>H%?iyQ@f7j_Xw&?o*x44Bs><89A<$mLx1)aVzLL|BA4n;=(LL$J{)j(s zNhv8&$tAIVZ@#LE3)r5ZV6J`@E_5OriWI51voTjYN}NH*JbbukuUOY*_v^F3r0dRT zysJfTkX#lZ3d4c^7q6s<2;h~@Vsw=L=9P3}X#pvL_dS$i-ISeN2Mxm_#pX9vJ&S;TwcYI#iRtMrgEcocp`s8YK<2~P;oYZ>tO!M z(TnD1({IvA0azDsMcr6Qf2XK`da7bzEOa0nr1(@+LD-=Vmi)sODCT!OoB_gYfH2zy z_S}jXp0p>QJN$|)5$I55g*tfz%B@Wjx!)6Dd7igRELwiaZzxLO%bC<*M8xn7DU_Cp z8f<3DoR>*jLfy6g4DDU?#H0BhS6x{cQ);29Zk&Gn(1(MdBSAWSQ+Y(h4pA)FqFg7!=weU zB68`XQaGCCXml({okI^$N0A{L1S=akFbGL9t3oAF*)_BU&@2%F3Er#NkgMWC0!4rv z`hl<`%72C9*vt{AJaBmhwo-HeW*$> z1`~ml%kIY8jKg7@LYIGltQi;yujb9+Fy3h+2|dg@uq9GW>NlW)F{Rn#rd>V zaq{uQLFBgcO1e@ny?gbXJ}_c9*Z^Oe=nL<$az2dB!?={oU`+qQjg-T}yc%W3Zd_?- zc(NUeOdP^T?42m*-}J319IZbBgu!&CzXzcCnO0g<0BmTC7sq*xFx$TFz;Iy#BE;y<|3C&z{KFXfRcHGQFXcd%l^ z63U+Ut!>u$8U>$%qv#dV8L&7;zF0K*>VGgQjppK72$9@GFOELW?>JJ=g55L+jvzFX z9O>DnS=A=gksB!?Yp?_xqoIL{oHK@j!C$c^RR0ZYL8CA9F`|ViniyV4oRq>1mgr^I z)PkfZv_P3PAf(m3Xm=+l!uy`sVG(GOw;o+sYVaRisWIl06n+CK?Ka`4Ylx%C-hJup zA6$uf^$36~IiNT&)D3b*-Bs91pM%MY#Z$^FZ^6#M6oVIahlERLlNV}87{-P>B^TtP zIP9DTSpb2V_I})=^j#H;zk$c@^mMp}qFN)-THN%e1BY5gx;>9m$XZ--RCSsdhLZA~+RKDiL705Sm%gP5>kTRik{Ah5J=<-5`&MH=v za2WjjRbfn}N2lBI#l6rGc;uF9q~{i9xQw+9q=*Bx&^*kjd7V>XmUFqGAINR~8{@?u zZ6$+PR!Qh85b1I$)3)DsYn2>StEF@YWqEWRcCEZD!|#vU4t-^!25fCjrI_u|4;US- zNco2(q$!GBzbVxXSzhSW6o&d$xx>>e8s8@i4DlMOWX0U ze7SUBX8LLvQjK{M*0vFXG&w&7p*2xI>)4v6oPoqb zM-D&0J-JXRf7k|X&FSQyIu?dZ&6?Lu!**U&>(sfaNz9<&NCs%R=fi%3;-=K z+|I}!HE%10o__6cX_YjkcEb@qb)40?#5;+Alxik<>(fWnUL?Xpl?(J1RFdsQRxB>{ z_K}h}8%;PI=v(RvmYEe*XQBOFsCsR|x++L_rvO!`C5N5RVqiPA+vW0uE<*h|UHSVi=> z@IZEd#%K^X=Gc_{cKsM&zCHvsFq%|^9W1X;?#M0>1nmV!E%$n$7pC z)?=ak$hHsIun%$*r21W3{XbbDs+oo8EGWS5P*?iBvDf$J`QOT zDvoOPArvlN8M&$~#GIX86g5LpQvP_PpqP+FXLu~ECA@^9Wn9n(#2`(B7ko6=7P%g& ztFBZ(N-1CPY*v9zC~4GQEL?j@h%-cJ;_(!FL+UuVIOc28IJ!XJjF>&mkU)bX`w$Cr zRgZm-ZIUStmakl-AwiZ@T>gVwZw?CU3l;|Jff*Y~cUjZiY?j)&YUy1rAjfU*e5F*d zWKx2pEq}OTv6pYH$ne1MfG%&_oD7xYMUyV+_Jh3+4@$#!Vn-X-!M}1PoV*ZgnJ(67 zgSApl+58q1_WVn#h`ohmvi$Xoc1SP&tVGqivV3|oH_YTS2};wm8@a|O)?@J>xJe1 zOIOlh{u20s{v#oh@GvW=f3-Ho9`W&s^j&TD|J0R6A1QJB1pm^N*xz*}7pKuA4oU={ z;0t@8P3b$ZNA9qG-#wyZLU@>@OA3C64A`NukR|364@jo!U45Mu80QV)9iV`Z30KA= zqA#aJch8&k#4kQ+bI`Kpr(k|Ai{x(F=NgwU`y2Z5(*O0bmU|4%{la_6k4Jdo!6x>5V07*}r1t(Ii&xoV!+! z^ayZXJnCk9N<+NNeZAW^vRX<&geHY$^-kQEd7@3>;lf2_0Jcb9bV_@&n(V`- zqt+4em$Y>bxkwnk8q1zJJFO4V(fIfFCB&7IM zbkyPLD?~6P;w>Qg!zkq$f+>kSUeJd-dy|2|LmxO&Ti;))f>0Kxfa4%9n>9-%w$*zZLn+=%eEzYrj$-C$)L(mIQ z935TTg*`bp#I8T#mn`>c*n;m}0p&N$)_?EG+jFd>lw? zB>xI(NH)w$d084LfyKw9jR~Y_|Cu@%r8Bt3nj!NGZzc=FuoPaR=}K}Fm1O{c6dfuZ4U>KIdY7$Rnrq~t>vPX!use1?HLE0=|}Hd zrgG>^IuGzGlGj(h?dgKK(nmT*igdf8mPbX45)u-)&2^4Y{-U&BlCpxofaF?s_%r9a zi=jJ*5@Ze4_|hnZiC=`P?-}zhH|_1uu9gAk8<-4_&oA-~2cpghq(FV897bGDNbGHt ziY0VvumYvLMw)0;cQNHO-VEirxc&(qU1iZC75N>I!puFv=+s|qhs?29bj6OTk4!D# zU6paUy3#ZvNS@uBN?MpidJ2D#>54IPrGL~0xff0o_mPYD8#1%_(m|<6FrBi?e25Li z?$4f#?ZnL(XpTvy^i52YNBNSHi^Cqog?)?x0Y6dxZ0EQ?rbXOK&F7LNz4(He@J#9# zwNxzvK4Ab8YYK#@4B)t>NP&0}PmEX3w~t9TKuPruP&Ab2ZbQ~_Vy9{Z8{f|MyO3T3 zEUV2zQuy#w_Dwch_Khv>H07@C7}0wczvc;P^4RN~Pw+}bRhy&1u0YW`1HpelvGiVU zqwd!u#@N0jE(TAhZ`Mlmo^{ULOLmYFzAGD<<22-Zel`4_}OYlv- zB}Ai3Zlg77Je;-L`FX-3H0Y8NSUjvP2+mRQv$5P%zsINIZRfF2CX#NN$Ym4ZB?m?e zzJv+|js3>FwRCZ}tUX#~oOU%$X70vcmjYW1U$*1@K8;?}?iK>;@DIe>wKfzX7Q0Yi zKPMv)d5Z1EqQ`+13F5?Y0N#Qfl33`!<2hPfpHS+KrCgYZcGOAPZ~6A<8Fy{;+>dc& zG0R8|G5+Gwy7_6H9l??nly$>-J)p?zJ2HOpD-7eCQX-H$XV>-Ekpyi@UwmBso;20T z3#3-kE*xI5e#aeuu@i(!nl+w^(NNe*k`OMPD0><>JGaQg))hLwG3=d7Rac7i5HIJK zbE5z~>g?5`%nStR_=^Q}u>)vZa@$JAO-qYcvX`FWP^p`qN|XCU+GF2eo}2xzJL?vz zvCIiA839ySV8Dbbl}mi~Ruk$eXIU;E9b5a%)LD)78nm|mS(b#z$GfH1dG z%T`^eZB@e-SsI_e#(sypW&buOQ#i}-{fHs?u|&qqjXN}fn7s9d7voXSxB)P9fR}yaW@((JYBY8cA zXWebAw_5`=as9jh_pcXXfd4#wA9fbIG#eJ;pbxcvO`RdtTeVRH3 zkYCl1{po7_*A*GZDO24gVK9DXRBQMxm#Te6p%e((?_jJ?7(YK}G|!PT4$fXrQfzOj zh~kPag3jHcjaDS=NO43kQQo-L8j%EJ%Kj8=d`&Jq3_W)wB7sm%l^v;+7W_0J)+|>u zVzi-&_fmQivHub-d` z3bY;81~~b$m*d6f0J=8(l_aB94??)^sTU9BHuXA_TkxQ7q`&tigPL5Z5(^YnO6lY1 zV~j7j^GO7yOsq^_;)1!yIK75jZ!%hs!o2p^+}J!2^@HeoftPmR@v}S;M}NR-mZ|Q# z?tI#-)Q(M!8Ca7KMTCUUI-nKTc%fnkc6Vu6pubT2*gGrSIA-jkBD z_#(!VN-p7m20EsmfOERZVK-Le(xg{E-klY|-%1vH%uN3y_PR#8DWl>}$RQe6WSC82susX5+^ zi%v+QBb6Og%TF1s?E$iSFa=1pym{Okovd@A=CfkK&3rrOrB`<_7Z-%rgsv4QQs%Ta z99E?YTy059uj;NL@rk1 zuJ^Qs#HjJJv%oFRs7f}Ym2!-ewg0w+avJxQqu@KPXYC-nKSjkTec(vZp)pumMBG=+ zDnqFK#M{=?60nBEtq_PdM+hp69DN7g^_X_TqPrq+XC)iZE@V}y1B4Bm{{?D5mA`AH zFqGnyQOM*&%JRK9!mKc(R`e?@+ex;J!UnStr9I?z+gt7J_KspndC5ejN?DO-jW@c3 z{`ex`5}*jJ=#&*!z>>?by@Jc5IcG+wu#|fDUIMc9SfixdQ0pf{+^7 z>MEv#&erDEW@mG!v$@rYMx3vK2rbOdsh29j{^|Zl{J7(yTtAj zcN_f2B|k+X>6XF?z|B``ieAa`mq7H-Ai`-&X6Mz3b724Z(u_*@a|>!AxN>1224>}; zb?1MQ3CZq+g{CEa74de)j9ian5xJF9M@|i7s3?Ui8TdgRTy3Sp=6tNVq~(CWkc;;U z2RB3{xmz*w4oZ5?Vqdv|mrIEKH)g~yfRJj0tUw&c+uz+?S5Vmuk-=k0D&d4qk_+Rv zDqaG$B?w)i0q~`Jml(P=kA5C&Ayv=ziefZfQ{J)}T|I)4+!EfW7w4SK2@Q!7 zJfzHyFo+hQ#hyXiVlY2Ovd)Mmoz_aLl-^VDg_k88>iPTB`cG@wcQ}ud`#1@WOq{@o zS;;&3Il=`fWjtR60j&|vYGdh?rfAIiBR~_4gDQ&Sq_d)I_W@j-P5XwKu0)>rE_yaPE>Kx`xNxYAY@JVn$r8-G zi$b3#U<8VFWI78L&`G5TPPFBBVSXMZB?gxjv~xscd?#TFj>AyKEw%ke3;S?v>PT9p zcIEX!s zy%HRzdM#b2qVQGeji6upt*GVGZ1CzIr{`}#g8irZ2!=#{NJltIUVWh(qVy&r<%am| z`Zx^vk^4%@Pn!_%W3$Jl4pnX}ZTb9h7OQePG)ND`7RApg%4Q?Ye^0Jjif-%aEyNpG z2;YlD??ohDCGbQ^W!)BGlVreHbt9*pNPtiP3uw=&>;;zJN&%*DK=rwxDzPNs6@RNi zStp73o3mEXx!D>SrTLv7Mwu_#71fES^crrzWdD`-E)(TL3_ z?CN%N35M8=Mt8*(Q#uLUOfAc4h6lvp^+K&w5d!{O8W#Bl&j}8H$fBH_CLk}e=r_WT zxHm$3UF89y3gCPwtwMlHQIm$E#1}jT1g8@MfeNpR#RP%y8YS`uxK(33#E&>|pSX=i z;}IG&-a1Vr$AU&AjK-tFo%_9v)2Ql#>qV46UEv;bMe_HA2{Gfi@o9=Y4?uy>ZM1unmh_}1#t|yeMT2DOncF~4TyP0X< zGCqA`vPAjwnF&}xN9hHGB-T%QY=;m_8J`1cGm_MT>8 z53xL>$sm=#dEaGAG})}Ox`Fzz#@nL#o=s(o032=ttJ<~h8+dcykvj>cFL#iF%J^ac zx`Gabuf9s`F7_n3CBayRg_1iOUEP@%b5gm=VJ0LL9<~CTKTp6)fFzTjMU1S36Gj<; z2(4K>^c7Bi!%+~^aR@p^P}oGtFCbg$8Y!|;<`n)B{B+94| z5+^u|L?}8d#wK46BI00~Ph45y4H=OE)Cv#;TgR;5=QH>Nc%Rb*?fr0p4!^tT@w(l% zj^LJ{2#-)kMSMoM!tj0|S15vgncsObNQ9IG5RJm1O>S8G3$&I{MgDSQpeVI~8qN)o_(16Xm)=!S|Gjku(8akjUAr~>$xwduZv zbC(HTp%&*&z%87ZIOJ&@j+sv|%l&T2-Z2V0|DN*n*!arpAx~IK_w1lyJq>@a0d{Xg zLbTmRIL~naw>2V3nYb&_y!^Tx8Sevd&X<4x{Pg&+zjuDJ zh#9GFa(*h7`DZdE&0S7MSKZJ-Pq6-Vn*njrk9{;oT{6+*S!njNaAs8#Qu=x7| zfd*w%akv0Q(g9!H>6;;@jpfi%5=^0ZxgN(lzMF^#%0$KlamQ0y09B(9VPu<#EIu=w z%qG4Q@?MQrm_*M5UrXKv7lGX*3UsJ=@BZF{fB<)^T&$DXYOdhKO-UBtA;JO(w85y3 zLVl;S0>L$Fnh_Z*?pqR%DjQ%(;!&Fo#nKuW+h#&FXR(Tw^4w`LWi-AwRxIl4p174pTLO#lpNHcN-$65l#@T z8k)X0LPyEjPzI%nfmH&rs?FPkNp#Q|Yne$cUZJ6tDmlp&Ku&#f9o_cizv7+h#<0q_ zP#VxAx)ug@p9NMmIe8RfpNxQSZFg2QNF*bu zoTNA@P`lN1rC!cYA{z5$HaPIQq)Ec%H^X~@`LZCKW2#GEci9pe9%-R+EZp$*N;K|j zOrtEYt`eIESdo^JWtkbjOnej5t`f{FV#e*@d`fnfa-)gtn{Mk$CHqO+P(yKEXvQjy zl#D)0cBMi;3vW5$m!cP11DX0g=aJVP3^AxTWad1UXXApw<6IK3W9!!7@O$QSMl07I zO7`gHWk{RGCNqc;<6c~f$!}>$dT3{Bv$!xu{zI2IH0Sxz(J zM+p}mMUN)d4U8-8T@{}Ji}dKjESiw4)OOS9*on zP-{MECbJ$SkTF8t7qnv45VWHu!IZKbl05eG9Fd@aT4`gI;lmkbGFDzZ9&o8dC#{lc zq*To1IAUPu+>!;21rkPsEDDDH)%fZvCcYaEU%IkyP=lxQ`wOjuOMuPv^g+6T!le~SyC2K zyRu01r3@D2?SAFGj>FXJm{!fkpMpZE@+Q1-$A||gU5Fi-Qbvr zT(4_oL)i7lts~ViSfRU@FABFvH ziT%u@oG$>~5wBk=!kO1dxEAe|K85PG-(reK~+d&i+#^}Cy!tB!JI{!+8&OSdH+&b{aFEv)Q5PCboyfRQNmyG3qK!R%B(C`o zPe#IVKxTk4#;ks@q6h%%EK|&VT}PsYexKZ?S-)RXXxzAPK=R56lNs>}jaGcG!dRDC%Wme}qfx%zq_Em6FT1SB$ju#s*CmX)A_`+HBn!uR zb%D#Ycg{`77u!IbL10et?|=W>QHqfX$-oji$f7)wUJ<22(whUK^2%5cu2L#O)JXE_dTTp*1{Bb?od*p}6meSeRBzQ5O@c~zu`_J}Zs2gE~d^ziUAdUz-u z?S*6T;}3A@nk*ECelWyI0`8wHE!~S^3-t?=DJ)9ae2FmrRbOqs3%*(06};)HS1U4F zmu8`8Ht0v8<~orGdS!eReZIv}f=;tRyWk4@VU;a37hA!j3EN}NV&S9@@bu-oe<{C< zfR0SQtZ$6Hj%SwyNgFLZns--z(DLT^=M-}2D2*y81moyRJCqj{I{#2G0Ji=qcz^j- ziLZtTAirq9r$_?2TxlJw$_ZFCrt8R@O2w?LuG))48VQ&^6-L~gV;v1b zPjOo4OlwdS>HrDNvnfniX$ChaC#RQ(J#;B#*P=0zjGoU62)07iUc?$fRoquDK1`9b zGV?}HDbZLiXZ(?}z^Xxi~#h zDwi?K>Bvj)`DM-&C_a-kesibEkj=;Hi!AnV3SPgNPGsPWEY55 zG+<-HcM1#EPEJ^ELW+dq8P=||dH)JCvb!^XFaFi;PIFHPUG47hQyoqzLt9k_VG!jp zzFGhTQPGA%yt|w>>L|Y&2R9~HGW)<)UAto==V3#~__uLJ7U&e}l{&;M3M_o%ARuY} zEy03ZebM=1wIxhJxZ$hsko2+$SHhwXywaS*`lTw<-Ki4G_>gho;LE!uk++>`{qGjgAnB08pQxSy!-Yj%WmiGYNNa8ubt2)qah z53e5bI4^sR*7!*(RZNJwpcLmz}tA&QWg5%ln|y2}4fLiF(9 z@rhL@^q%c+Cly|*Z*9m~T*%5LM77;ApWhZZ_7vme1OT#cj8&%b0A1sN*B@swob&w5 z<+~`O@0mPnr!hx@Y$N5l3gXpY?fhi}v zC@BKHK_tszTYTxs4$ICof|cW_=(A{$^tm)+eqsSl`MMc1lJx-ejf~*1N3(&dJkE#x zx;XkV9Yjfo0>nC5v#bcG*=#jIBiolZyo&p@tKu|j8`XsczUYTb&=EupDn?7fKFtO= zktsa&N8#eg1r7U!cD!3b#|L|7PqoT49ebk^d8U-MnQxE&ki;n$ zCGs@3IA;N?>8r#sqz^YEJGQVRQe#zQ>-N#Lv92&nNEZTGy2o^s{K~XC3KLaFmE% ziw0okf3~e7;r3J;>nPhD>8h%hlI}tM{NT%_HX+#+Wuza|!GI6!RpDQT5c=^)8RhS% zpNtz~^a^>SJD4ST8>EaHAPUnfw7b14UyK>)GZw36rBK6nQ$LRxdS%QBLX5?Ib`z!j z3FN;meZ7j3upcJ8^Klfu`Y*OY$FDebVH;uaYIH}l0slh;WU)5K#oJ5A<|wWAjhPlZ z8|6>4&Ur&E{=eSSElA#6^Mai2Hs$fZyt2!br+!K6-6$jhR_@jTjUm;WWpk*k&6MM& zP_T-NV7_9RjCBLbwooQN&4*-EBw5V}uboE|BGnR1F!(ZiKF(20Fw6C3d#PhdNdw8% z+{o)W5QdS$y9Q@?L~@cbhv2_q-lIZOhP2Pb$a{16Ro+DJqd^vhJ_NEC^$X`p$Xn7a zxmHdKFItH;Vh0;3hYu;d6~V!E1y$mWh#D^|6<%)EdpG`$LHkw^Cli-36N$1#WH^x` z8&2z+C`FvS03A`B0feX-LvFVSoM>3JC4CBuE&eWvejSt7QA|VtC_S=Lq-7Z+&5E9^ z6gtOrfMPNsv37F<+w?d_axD}w*l0o!z3!od!*AZb*+54puTMA74}0e)8|d)-{PcVS z?VlfA9_{ZPx0UZ77!~t^&{s?c$GoeadId^lR`0!SS|GVC@wKxiw1mv{TLgaOEgevh zUNe$r6g=W^%&~Nbt8NUPblY%&@oG$VVWoio*p5juB8 zQWa&1Um-sLHQ1$$aCRe!xR3>TMQyIshk~+A!QavA*&7!r+KYC-mtk!^*`i(pi%hi| zX?bq>%cJXp!iq$Oa)JbVzzEKQp3jeg*>_QT%g{=yQ3b@|gQFhs*>z|T`DO~ug-LEF2VdgN%WJ6wm&aZRw8bdH*YCcZQR{_3J(AiOE@*(|gV zBA_k=Jdz5918Z4)+0aZ7A2qnZ$?z8xbeYBGxuat=#1n!rlE(Xpj&nR9a7w|wB%kXP z^LldIClsAuTy|a`oouL}oHFvzkiHiNanPm8;ftf3DW}+ssMKnID|)b}z=`)=Mgp|P zqTsLV4fxo#QFfSwa6>J&SCQ3MxD|pKpP=e;jqx|{&W~lB&0ePiNaGAYgz^D9JGc+C zufdeHQAf0(Xx`oxqwzQ|GXJw7;Uo!i(Fx_OG_r&xCrzW!0L~EJkPP-ADi~XyAK2$m z!amAW{HH|wq8BAWpn?#fIYnaR$(=gWl(yDz^XCgmO~gTwds={~=muus&TU4o5_R~i zJ#q5Z9kzB$wxu+jt+#oGEgNVAK^}7^N!M>OU$e5Jv>Q|rg=EVDZ$+RJIJ846it}Un z149e`Y4Hdb`--!lhjB25GXwWvn!3!xL6my=-Pi~rUMU?ysSrYwM%(i_2L7vBC-^DLQbV7$ZdSYaKijXJMEqJd(F4TwxdI zFP8lFlHYFRx3xupol*ym(wGlNU5v^gNH*J4Trh{`Rj1`CqpgbM$;_!gsHkLd={6}y zlVW>xQ#2by;5lR{wTl>0vKSN+_NQgupg4_8W=}&RY_MEy8b)QRRS0}k8kVK=1A{YF z+O#+my6_-}6i^7uZhZ*fEi=@ORZ~_yb2q~s9fXI4`*^-8H080`8^}<>p?r6wBCYXy z@1l3=BHz`jQc24IE|BrYuvpWyRx}UFZ%Ql!ug zJyxbL$klv8g>hKSud_7i`s|8Jv(`-NOUujE0JxD`(D;~#T~L;nPrxcDfB8>K0XdZ= zP+-r%4v3<2vI1uNp8*>n;{hyLfF%pCWC4~efMNkmJN$)@e-$R=kw?EIQH$#mokw%i z$qdv+o`WJ=L|`AktQtZ-^1iiLlmJQOYv|G!^c(1trd90nXqQWiE2w z)Zk9riys!=S5VD&G>racWZtf@O7g~uiCa88&M6CU93_KV)@l?b=T;?%7F4`?39m#N z=KZsGJ>>0fdd54MBuudl@)|Bv$)^iPiIwu0jwJ4s|AgGi4ek^wa-sUyZm_AB91kSS zgC5#$cQ@NFn~Jj`Mr$gVw4hvO@+)_(!f68vnHjS7;`jJY_#BY>q%TZ3tdHdzduPW`lJv}(=sRaBg(Hc5>b8>oq*yoU3^nt&-uriQ6PZ9~}>LaI- zkSWNT5=lT2JwHY=3H`%PC!t?k2_=lT4JjpRZ#Av75=gqnRufAg6&R!@wS@j&PA=hC z*~S#vQKeg`^g@`@NngXJRxE!ahli1(hV^Y2^HXU}f4guwEdtqd2^8=J|tQyT77GG#Vl>08QjB=g+M%;n{*_w|+2ETSBH zB2E|YUcNAreu7Phk+Hk^2rEvls!Oy379Dn)X5yR|M3rW8p{;UqusTCs@BzHK0|}(a z9F>H|Xm!&)^ws89UE_2+vtI%mAb1;`@>P`6fW|U6zm#3vu(a)5kkVNZ`0|M55hNHB zjp=0EZFG1m8Cs7^RbC#X;tVkQmOD=N@|Gr%4AQ7lR|*CXqYJA9+CM+wgjV?sg8`!v z@JGjcR4qAz18~wwS;vv}N>-|4Ib`^!5y`VCP*%VNFV{mO0VJ0((3&Yvj4N_Ih%gZ2 zm=8xxDI6~Ag<^wg5?VwX?T#d8J|uiK@!^SSk7ZyKU55o3PDvr)ZEBf=w++F|Op6*) z98@6<+Cdhwc1R{B;v5)*)5?gn+bK}c7I=o5_9S*D-G5Cs+ud$^bA=yNzF4y&d91OB zRW?}(C4y+m#;30PwFu;NbS=ii(W@dWM2WI;lBs98bfI#bKps>k_9t5f!JTFq3G%pz zA0x$g9LHBUxX~G%ix7X3!z)&~oS+qSl%N1JX${^7*-R!Qtb$^tTPd;Xs`H|=3p=HR zWqwy7&RLGAHklXH`ApL$4Q!$Xk5U7YPv^)2U~dRXTSXMDiUfj`m@hd>o7!S@Vtxg2 z4y3LqxuWAFM0tjSQL6%Z*-h5+TEe-%~JX59PU!_ zt24KN6(DGCOK4pqgSHWBOj9wD&!9vCiv=xcg-l2fkX*=g(rVW{x7RfnT4_jBvS_#) zL9(^#8R;xl}R#wozzy-mkKRSw(Bnhy&BbsnZXA=j#aoQroh}>VA zL_`c-M=^+4hfRb zLa=U~n!SyF0R6a#Fd%b)13}}&tB5Zo&o|MU)gMT_pr=(@+9GAFK?$>5>DuICfPMw% zHZWrniO6^65QG&mPVSI46l({ZiPG00S(Ynclk>w0@-Cx} ztO!M=D6R&cD$R)4FXoI*?V8;3(=y5n=PX6D(4Lvv3)mL@2~@PAVtgmrYU6@NU`}2N z$<-JrgLWC7s-?K-1_&aCMiGNlIW$v?<^Dmg6uBJEXegB+7~IhW7i$i3)ws-R>BJdO zWhC+ngY!?2MLEeLEbh%{qI4^;$RKPJr83~k26z%>1j}2mK@kULWd(gR&LP4m7kSwR zdQU`}5u>1DqF}|U+b=O5i97*Pfzk~zGRVlt_sV->IWvilE0ZK=rtBjJ5fMponOTc1 z<%a+_b&$JTPdYGwdZj~3XcehdOAxQNc&| zplcC1eswH;bd+q!i>My@Q8o9|gC>CW@JaE1rRje}Y-}aMxwn=b6%aPJ{ILOHhg`c@ zY4TLdinqfm&YH1;B(1VSaza2$b*&C7$w;Iq&$V`~Ex^hw1Gf zYzHS7a5%}2N;e7hlnQi3$7=zCFSo54TexMCx&2VohqGw(=u4 zxm-LyHSFG&mbAEh`W|N?gu%yo1ggk8&SV+;1P#~Y_=4m`@b7))pdH>NcoYR_gp=r+OFL*1VKsP> z(Cida_nP^t%oygLp|uxB4QGHST@_^RCQ8B{QjEieigX9mm*m*a@(ArMf028kB>v87XSK-pEL<2I3BrkBLzPDDmAVaeJgdP)o zLqe666U+ldghr$?|LV)wQuIa{hHXfK8wR(#0wpWTzCooYCbubNMXW%5>0k!F(-#G8 zbd;O=o~t``tKV*|w77T(A>xX-zd5xU;@l@4%TA@Gz(@o{H0(tr1Q`$-r+yIpl$q#B)ff6~|E$4U1H2+0E^0w43G>0;|w8ol}j#=mA^z!hco!{nv zneW-$+}zyV*@1sIH#f`wzTAEJ1ZN)=_ z6;w+^1Fs7N{s7k;(x&VxN8dIr(wJUFNvB{4V2c0tH&uiUb>VEBwB=7ZkOPwAC}zT_fupAOkZKD3$}9PNp#bZ7 znic9DDZ;+TcMJs?X2Uj&`ZdVe<6MQ2QA=jSu#-gZ&X3m^7%fWt-rxHu!^wcugmM>i zj#TtozL@npUROkPZ^54;&^!Oe$xCkRk$2}{0XvCUlpCZ(O7e2P?1cGXL)UA`ZlY^- zUC)qrLPK&!v)rp}EC?8&z~g6dmZOu?gTwyW>G|cW7i%WfnwMOaNDOBSg%tMV=1=WJ zg7n}OcMsEd>3S8=qq7p2yk+)SV56Lj08WV;FsXJppZh3%SY5Yv_=^JIUft2^(aD=0IxD&cQw||Ag@66)Ux5(FU|Wm`&IweAaI1?@sYU&m zXH`mtxiiT|Q395QsNpeNe9BQ9^ZDfb|2*@GdvFH0NSIRvA;=lD`9cKo z@4Z58ux`}vGfH!KEt~cQ`O@@8x*`mPvfGjn8LWo49TnU z(XHk5KU_YSbv~ek&u)^Vt#!FKtE*ymSPNI7M?zW$(Ef=3sg<<0}24ClhaNPTnzFA{T4JCv@xD0 z^=cviKFgz;U*)k2K+R*N2Zs3PkiJj2OjHDAW^Fc5ycC*buCybL5lp zbwPWYLAO;a^9>u76f<%$rFNOoO&Dd!Pl0KdiF4av{BjzK6s|U+?NVmqQBltS7I}N{7V?kB(wDTurASZqRz6eAI((L#ZI>z+ z^R3~M^&lUg(Xh<#Cmm44*P<6ydgkVhbg#9Yd;?{SUhGP<_k~#$)b|2kjZS3gnVUCK z<`$itogaO_cX{Yvo_=?DV$O$Mx~3D}!oFBHzPLOtFM_YR48}fPmQ|0~Z}fYzpc z5vFH+m-k*F&y$)Mb2o+j%bMWKT-(~KNfYj^mnn?K3i*|{3(ARNtD9vMOs}Avix@St zKLzIbPL)->0>k31KuJl{EWBRI+I5^pW?X1vu7F;khlh?bpyyvr2-JIy+M=F*%U?)P_wGHBRW=c7wgF=c zcX7`bR;TTs9BHDyT;Xz2$Y1tiXv$L0D;Yb-Ts8%l1KF58IiOjjveKWv;Cn3LzKG7t z8(%=yFA+hk6F0&~(x>i1#7~*IRcCGE27i0A=t&+vJxyyGqGZoCZ;eYqc`7!ys24y*x#cKa zNj`!kIW)*eb?;aAu{35gm3FWZa1x?5b2pG=(th*0e{s0~?)>QTKmCKFi@k4-5Boe8gzur6EIxMENo4OJATh2cqAg<1Nu zM7=~ZW!q%t;i08d2=|oU%Bmm_fEV?kSJ&0=t*>4_Py5Vu|6^E?&eKQp{m<6+=I&1U z{%7mUt>yjCb37~PT&w`=zz}gJ`tscAXJinuJiD_F4Mj=@F}=cZJHUA!leVZSqhXxO zdb(PgMM-{*y#HbyZ=nVh8M2PL74!qdt6}J3oTfC(CFiT&O5-Q#3{8sQjwQG zO2)UszGkDviwKK&@!Ie&RWTU2Y{v@6FXas>-Nq>+W{WLgtStn?SMU=x99o&cue1;M zBtjzEp^Qrc6B%g{>hc%l`kDlJ4|yl_LIS{F7d`~u;jd#Hd-aTg0S1#^3)UPnGMbPK ztok#8ynl4t-5r0^L+yPd#7X>8O`hfCmoCIih8~%(&7vt;e=oWdtS5-`RzGxMG5GtMBIF9cSpE{8U9j7V4 z-ERFrMB7k4=RJU;2wjl?j~M~iJkzyN*pEgemiYp~{amfmh@my<5hQI*AyIrvg7tRm z=o)3h&P6n$ZF)r140spiRsc+hH3z^;XloH$m?=5viTbGVDT~y8p9SCgM#D!h0QTOi z*(bDsQDZ?gQZ@w=a9S_t1g;>E{(mvA)RyRU_T^Q31yd|KV>bB?QT{-f=>Rx-qX_rV zhFOJDo`cqlp_o_}e=9=q0Rmg#fFSR`J>>PhMYm9SgX_3GGWb+2=+&51&ZTJ=Ys++a zCHgSMK)t4|#8^S*g3y-f@{fuwrLzI|z1#FrUlNu-V&pL8(-7eKIq#(r6C935W5a5s zW*Z2nLgAKEP&*kU2rNvgpaLODfKh;G35Mlf5e`bmfE!(b!Dt!@*#;wZ8j8AgIlVVgh7h5)$fY+IF`+}4T?)M zMJ@KyR1pkqo<70@(#|nl)>o}ICrbTze~&m@a83pUVpQUQ?8WhMl#>j{$om%?;`SGw zqgl^0R7+Jj+~RT-5Rtww^snxO{y@!(bgD!y8z*pk6c~|slh@r1ngKD1+Dm@gG?wx1 zJAv2cpbaLoj%vze)*kUml}gRE@~pV1PrVN9#+R#d`q~m9Oclyzowlc2cjZ{Fk6wny zlnRen$N;?x!IPC*5x)R6Nzp*iP@m8A`9CFFf!aLp@*PAy6x`YoAn6t-}Ms0oL!)jA>ATy+SqRBtu!swIVHC_NrW}Q z(@x^Ie35VgG)8AMM0+w!H^~~Nb(rG1DV>5OBMNwDvc-phkU7lS?-C_Vr8!otDu4{M zC}L6?9z-k~!*$;`<8VM439O@ZEs0PD=Z$sj1u$o&Sr@H{q0^akJ7?8)%L`W*Zd$0i z&|Hf&y4FFuB8FHlobRU6m55zH(E`8ZSI5ERB12qeY~J_HU^1Z;X6l8GMLYx8s_s## zYB3C&p|!@L6eH&Y$+I!xG!Y2uqfsDcu8zchH5-XtpBIlQ&WD$G={)#5lU4rsczaE= z5zb|d(X(+Z!_BZZI!e+p1V~v6qktd&l!7$km5X|)eXpY$l|MeLPhqpIhBObUJ!(w* zY+;_Y7SJTXx%gDNlqt7}DsN2gkc|&i*LeB)mbu*v-R^hfj={F?y+S{? ztk=DkFO8>u1X=S6GvaL^Pm7vA83ouVPwPL*03y7AKwbZa+|?EViMH=-iA!g~YP_~H zC=!Plz=2RA^~DC14q_OT2vpXW3$j`eBEQd8Fvc?#Bnio)pilX)8Oqa}9M4b;|2EDD z4rc*C<{=E|X!g5cGcC?0(A7RVhT$_-h9a}e?}j` z>mRnCZ`nj^Lh7UsrM)bt<|v7#d4BpXbMXF#sne=bhPL&_XhU`%jI)d+`CFVqeFK_G zQF#v2B##ntVn7VyaybbYj6`~xO3=VPE|uoqx{k&hXGhDO=##~O6PjEvwP89{ti>d; z`s2KA7^|#HP;0-1g9#@Dy)*Hpc6atPbzWd&RL; ztC)CIJEVeZccYcgrgc?XekLo~+;sJw{pP;L^yc@rSzA6&_snJgnbL6X;2+cMKf60y zFJG4JKf60y%lIG9^3-TZO*u6O?_J0Q>@m*Sdqy~4kG~VYm?hp$3E^SomEeL?Inji4 zZ}3dv%9T3KZBS4(=MmS`*Q)8Po-moe;^CpCe^^+siZsC&CNlq(Y)dA?dc=q;iQ^EsnN_mMsUU>a zajwBFeM5vQhd;Vyd~swZ&1FLMSa#9@8G==sEPalWDBpuW*WJw+3Z2@v%rd;egW`8h zt6Diz+h$S1O3W)1AXd^~)^noR?6wKV(AJQ?PdL>2tJJe7O&%}g2fUOo91<0WqwLRx zv>`Wpt&Muyw2pB0FH9*i)41GVsGI7rAy&E}k)&3*((0y!%W7k%DpM_bc<4r z#fEt6hyYO2VAYwF=F@?ckU5MNXqMYfJ4?IM>0 zu^mAc{8nLw>R-3LP<4LBqLi446X-_| z;svpxClmc@=}p&%W~7fQ#mFuvZq0=g^$i%zCp1xmqW7_(WRsl?qTU8;u8 zS_j`-h1KpRY6{mhZ8ksZAvg;FB$@DG7w*S$Ntn3s5UGOtF{K^a85^Tf_2^2b)wst0Q42gf%Kg}{KSGg#nemR$P^=N(y*~&i5rx^}& zS%U?>Xyqx7c?I8WLWW&>t|PtNQO$v97ppzKU4SflXg>n7xesWok+aIFJlBy_S$QAr zIC882wVnqF&B$KNH_+W#Mr8_-3LHmoPEOAc`+LXdhkFPA>7SjS?jK%U93E6)0?Vm7 zSX66o4==wxt=2t0y14X8MUVHsIXrfaykTt(XTTG){P$SLcdhi@|9Gb(bwLkhRzQ!PT!sHHz3vxt6~~$Se-L0Ncp=Y`gKfRM==36h3b&r zogDq&cZU!J=49{f;e~UtP1|8jONlwQW{ zdy+-hcU4F(zB@YWUmjodzdt-bdi|e;SA8Q0K^CRCqwoFmqqEBex~%OmjwU1_jGbll zN+ZpB`BeO=>XDPtsM4kV_UZe4V!5ier;p0>H!6G-=j4l0(|PVj8Z>Q%0^3?7ZFYm$YdF=YjZAdczmgGa~Xl!{?UV>?PQ~ufT-OHQuXSpfY8*OR|9+B_n-NoVg zl*BmGa`jYzKR^7x&@P`!dkAGKR$;sO9l0x+ZyE-iYi5=iS0*6NY<`$EShj{$JdhWm zoPei2W+eL#Y;kpX=pqx4_hhIqQVl(IDx-yFO!e?N&X|g4;hZ%qu4wbjHL${*Se%oX zV#h%M3;x`73e^&B-q5@AV|QC_rs3>VE9T~8eSoRpf&gBtE>1Ee-8RB)1mWEa^pjm078|RcFTPo@P1WG!+L>lyT zGl?|L)iiC#d_1+?tWKd$W30wNjdN@JmR_Mtec#e6)NXAKWmy60Q)$bV&ZB7QJo@>l zxt`X2bbdXTgMd7NmTO9jkMgpeU-2|cYtukl>X(-KrKNu92@Jrj-S;F`U^el8F5Og# zb6G$+<>q}JujOcjZ0a#IUC-ex`*9UiQ;_|b^JGyK)icfBOiHR}oW1!i$fhEDX-4*E z)=+(fFd6msUDXH~b1Hn6K`)l-o@LOB1ynr+piiUdS;od##>Qyc7*DC%nX&0*^o&OY zQZrETXa$d>t)+Trsoq(tcb-ALW9+>rQSO+e|9Mn9B~E2-#g3cPd8}ff5vnPN%QU5c zv*gEA=uAQJ;%CQV%A9AJu^ALO&o*Q8D{-cvc&Wnq^Qv$@R;(# zlV{ zGy`8K0G~i$zgaBRNqc{Yv%Gy`WR`v+70GO>kp)yDQxpz`qQQVkjB+nczb6?J;S(I{ zk*83LJP~wrt4NB->b7pAgztckM!0BE{_N?Zc3$#QX})q~09nf~(6Nh3KMe?QHqiT( z|6SL&S#I%;t=+6cyI=fX+j-Vh_&VNx5;^kU8(-arbsXZ<)|Q$;@UPQBfHrbEiX8yx zZ%xdDBgOqb41a{vhAj8MlJj5HO!~I_Dikbo-d9SOQ}eIs{QU;sZHVK#;v6aSOT;>- z86MP-=n2j`adf5d>sic>wM(Uh9tP*wS#e%ar$1=Kx|0I zafq%68sa1*oT_3b6%6Q$P6g3|%=@CioE)10Xc^n~-mjd7Rd!|mgXd{MdF1L(>_mln zp%m|*e3=wY&$dWZ`Da}s8pz*!fjknBM=lQ)Pqwr=)awyWVSPO1@~Cueu9A65zt(4) zdtK=&B;g{)bFMDc_3yp79tX!$9lU3fIe5geyCChYJ9D3C{Ed;9>qLi-5CK{#B~ZQ2 zu`WC_dd{#n&2U}ppH;X}CuoYb>Jfm5DQtC&m(fCK3q>8kP?Hp$V zlJn1Iy)>ga4QSj$m-}b+y$&d2QEGJT=RsQ6`QJDHzVrAF#^JmjygLvb7^Y{cr?b<8 z{?VDP@HDSPs0MmqdhP(Pvo|dP;xgg$No;}SL zo~dW|ghCa^ek|?8GT2YHdLIjV%~D)o5yBPTIY~m2k*uCoeDk`0et2?lcz$&9Mp=V6 zEP94{p7uJO>8s6$e*H{!TxU=Z){mk})otLD8<03eMPc6P1;%pPznzba8Zga&+=$u`UdeXqKxg z<9z)de_Hu)rUG1rQyClsurKPI7XUIatTF&=v@x;`;Yl*#1C}&$&Hf^`ggWzww%Y`6#dB(FQ(p5&o*e+O{iZmUZ4HYr&+N)pKafB z+VsWIH3{y5n0!m=4J(Ev-&)-ml(ntwHx?S+6PgcJT2}@yLYW}@P~f+%qhJda(a!mw z2@S~wiAj*t%%&Z2l#FkE{@rJ?fMKn4A|Y>ibcnJ6bC(}TnO519tI@Rm^`8H=AV^>=DzEG=hP<+Fr6u7C1v8fAzCOtAuCv`Ru z@~iWEewPyDrIWxbt$5RnQ&-5v<^xPQ_C#+jT}d#$XwUzm9&LuJZWG+IfN5i!jMWA4#fTlIeK zLt~n;#WX{Una5`)cKk?VoX=Kcx}Ca zHr<$ceA+kmcN*K25X5!%A2W~72*CCnBb>>QM4_VK8!F7UWyH`Z3RGI*k334>(ohyM zUOW#AE)V8*n$cfKkUOFiSwqytTq=g5jf$?p>MLMTb1wmlT{qmVE=Q{`^J!p4v?%gjanE-;`Gpo`Y&aw-^}50~Xuv`knhJ6J91($m^iM4^ty zGF#PUn;Kn=E1J=4scwmsdaE?wpiQ%u;H;{rin>nz`Ik@d$#4=3N#>7e66G|Dl0hdJ z;w)#{_Ae)+G{!k$ogf~woMah|30Qf~+Q;VR=H~9s4*a{hxmo^qYqPuim+sD&FL%4$ zot>9kf7#sH?ta<*3)=j+Ll@7Od)%b!hwM zE7p!^XVPtrNRBys$;lJM-ZFfCEBv!4V5?qtYKQ{X>F^~fFkTd0I8O8(2SuBmr7!lI z3Rvif66O%2rq9Vtoc}bgdAg*U@xFR3*WK<6;D21$_<4cT=uJk)DeIvhJ?|&9f;oN7!mI%sUva5i-qi|>T?oU#y&&L% zPODiZ$gILA_63QawE9TKhH+wxqfA=3R5`MW@~MHTD=|GY*RW;Ag_x!ih)RHtwey{MykDoK(5KzZNDMPEVx~?hSt#sbhW!flE55Hag~f2qSxSazivj*AB{u5kv2zI7T#y@6daiK|F*zI!4$0Z}<*P!Rp0fh{S5l zDT;FQ2D_r*V>u_F2Q%Wi6rZLM(E`bJC`KO~+DQwwCN7OYRPJF+2i=O7@;L!+u5G@b zo58MM)ByNmpw}&)--q*<|NI}})Ny;=#1Bk4|2MzfF6)1Hw_mnZEybH_Q8f>*bd_ z%l-cxPf3n1^XT(!fEdRgU zeYw5d|IhK9~;;`~BS<%lOB22FfidQJl z#)k4(Iv!W0H;AU&Gbh5;zrxE0-_Yeq0go4$(GjrPIt;@s(x4c*Rw@x&6EcGgd0%92 zDI-c?sIH>#`{vJ2RyGOX~g_2yKb)uSFndUVXkgw zM|8ZW4-QxgKDaIbaXK1X?Ks?l`MK~kfe6j%5Pm*2#xh{4IRm0glYKv9oSq6CK1n8p z1FzbC?q=p&e>dYR66A54|C=QwCtx%sU6gzsXtXX7R~_BhD<3`!6AyLv)Y7+vg%pLzEGkl-*TjQJ{B<%|Si%KqIwOeGDicgc-1S)^jZ`!WhA|*OvzYO?8qy!>x>7M> z(oFj3tcOl$LMmeR+(M}AicgBR6^DrOI`;ou5%|+0CjOEM{D{w7^gqROT7HOU3jN>S z+1f18|L*peOaAX!p2yMu2GwDiB)iBaUr>xctAq`N(O~LCo#c{wAE2tTQN#ljuSa(T}?^; z>L>urUP2-j9n#jNp0{r_n~!7#lRSls#pDbJQBnLmGzfD{_9#luUFvCbl^SJaC44)* zQc*@$B6$`C%(_cHQks~LB7NHfo$q?zRrhHw`|XyW z@#??SaibxCU3M0;0C$Z=j0DOpJ*~+=Il<5r1A^%@_zBw+%s$p;>jJx--_h2f@v?uu z%l+m7e!NT3hJINvB;hzFnSE_*y%rwM?fL{e(|f7Jk1)Ebe0z%if^in*cl$KS$*mnl z-2AX`;LI6d7*dw+#Sv!2m`>y6$Lu@JA_~KdF=IBmE1#ysou1kk{A#D;IIiUKflp_} z7kkL^deB_QD%Y@GecLfyNxW!nYXEct2y9Sn^SmmN9ntk>HIvU|}&+?_6(eb_# zl5~ca`0}=QaKnJh+Xt0%C|5U36G4Zxf@OmrHNd(Q%kGxk36~{=j;zS2E@hN>hIyX8 zu~iR9=VR!U-_8c2d$30SWE*+kUS6K{&rZ)TOTA6Y4C%m#BWvH3MJF41E2QsgDoz{J z!xON8dwY0!ezbq#hF8R3nYN?qy>#kCL08s78@h(vg$9J>42A_fQ}Y<^IycFq2llw} zh;kE`N7WQC8xCu|D=+FeB-F$up?YqmyEOxJ6C=HSdD9rV-#hszEulnUuB0f+N9i@2 zeejiRY2vBmOnclorvOn+WCp`C^QC=6#?VZDCMuXI(2QQ2qftKsf=aqE{+Maa9y59) zCz)DSGc6n|koTWhY5RoFBlN#Bn*&U{|9`n%*8lEoE#rSb&vRdm|Ke5x&#Wr^gf9M{ zSd+Y%9e_)C{iB%$I4|%QpQXAB{Q9*w^L2hh6Z2f|*EjPAx$2*9fap0cmKOwRRx^NU z&40#50doxPlbFfH4F>94p51(4hBp2<#srVx{}PhFr`e64f~Kv36`bb(|MJVS|NrjG z?&gyJdyeOk{GVI{LfP7mqfwL}C4-DG?K@<$46-!eN=00E!8@Fo4Lt?3=SKpLMVP+I zT7DSubvW@Etg`unb)@$kOs6o z=`(acx93H7r>dHqEeb&6%-1#R@P=;Eqgez3oD1bLQHWvI6`OG!&v1)=Uy4)ow3O`d z(;#Hzx#$?N$yiSRj702EVVf8vYz5&srteXhFyQjzv1WK{soTqKC&hYqa5igEMch_U zj^i5C@2CCDHuEWrIiX|1*05PvQT!HaE-s|I6;qGXC$gJdZpz z)f%YL{XUH!?q_8WtT6KN?1EJfAHhDj>b*N7EZUt|G;Men7d9`}7>#4=5mqd7ScI6S zb&m5+pN$KOCDSf+)AUSsYuxD;FB`4s^ZPuHKmTRT#r1#N-R-jf z_wLKxrT*_(o&~Mb$^vn*eA)rc2xaYnjymPzUpDkaT);e_YnDJzRGWbS8|L|1F^@DW zN8wy-iACj~pGQ%{T9{Q)Yy(NcltzgP^Rb|v=6pa$Nlvl}*2k&(ZJ(r$tE1VwU_pjc zav6;X9qXgsdfC|7kS6K~?m=jBcq>d>ry)myz)KfB56LWiQUN|Mzu3_8n2E6bs=9vB zn^D#BD9mRB)?}0n^s+QwEd;_Z^a|!edaabK9!k{3doCX`RHs{k{y1>DJAV$MGRH&% zFKcQ!%r|q25lgo6vwP;^|AgXx?ifH*_`jE%yF2ClPurWz_&?9`l$j_w#fsHav3_32 z4|{2J#4d9Fu{K4lY9o>bU8!MJdl%yrGy(EXsBnav);O!FI7??|I3Hf#rL%XUDmWYG zY^nmek%!eaBnbuug5j88X)DUGgCncy#UMch2EkQW4bW8qzUfMEDaWfYS9#-W+3VkR z9;s?I)}88#*eL-Ns9cPle&R6xYA8>t$y4Z=Nq1M{$WnAA98_f5A1=noQZE{k`#3px zLk&E2YnI0nl?y;8<+nE9+E`V==Qrap*Bpz*sFVZMHoMYi<5;j*HH2_TS zmD1LM{HN(1TXhT?=6PD&Vevg_U1{kto13%Z&*Hj&+N>6|_Qwj2>p~(y_U#!1&V^p1+)e$4`O%TI}P7eO71$X51yuwlPgYyp-+!kk35w zpR{_KYVTWqrs@B;b}RY6UhZry<-g~6N^DQbT+e;0|52^qJ^{x66VKmOFBOF zMkOoVX&y19#b9d&bbE(2%`I#zIBVdoxga&I`R!IU}am1#W8J`ZqqmlBDfO+`o^jO2xhRxoMASdoaLg} zn33GNEF&`-lOD4`Pvz?7M3rRyG@}7wj4(l5@HnSm!*`Np zH0#H7U{Rw^NG2VYhjg5y_c%*L*)$4K_32D-)(LPhBw|;H|F_d9{JJH&n2;oA=w4Ag z2~t3lgam>e>~3!Dh&~=fonfA*x`sl8?2qJu>}eE@M4)~4wJaLbLI0X&Bb*Cj9VKY> z#}|1tBK?>KI7X1-0q0}(6MB)65zR>-hhc_%{d3Gn#((@eCM-v5FE}Xuj9in9Wb3PP zOu*6H-l;+Kb7>hNp6=$?Rw+)_OGS98uS&%jNgTyq+J&yh*Vlv-CuckpkoySX_+QG$a!nvhrB5RLF{pGCisSKZBl zM#l)JNT6z*4l*1P^rDzx{Z6j|9e7ZF`C5NiT}44mnSS%2df-eD$&4mqNVQN5?yJoL zp$OrCFZg0*1BUe1!EaG&;V=2x`qm#sNk1mZARnUcP3c=eM`<5$hPFzdU@?l{Il0X{ zY0Q@(P9c~AM@8osJi#o;qBOT&-}=S}s2VV4ml(z3gHaId89KkDWFY?>MA!1KG#Qj& zo8auOZn$B@K!J%6^4?w?9ikt*{_aoz+x85csYROihXjWtL*G)CqZdQUa-$UAW*{GQ za2gq@WsQkbO{JI#T!ALG(}5id7Y5s#o5oZ=)V0nC&Vu0rEgKcs_^N*$#W~632)G_k!f?FLC_%eNU~GyQc$!Bj^PyA+T29jn_H+GPAj>+*+tu%Q;Xr`4!s=JmAbkU zOG@BGybEZOXEZje;4oAu)X%^DGvEyNVVEz*Da$j0N3s!d7)@&f>iYYlL*%97t2hcg zv(}tjpi{R8ie+88IunsPbhfgXUg5YM;5?5DmxmB}(2Z9L6h5A5bE){%IE zF-^nd!^096P)9$v&w&{zpX_TU86O{X7P%I1BCuR)c)ds+iU;R`-Oc7 z82j)`I{3iPw-1Bp`a_xwK12ci5R9`AgWo=|$$tcRBQ?@~nn zoSu2)|CEMd#IkY9PqnLYI3Uw9zldjw{NLT$-Y&=g`*LgP|M@IWNqw8r@IX)TH!#KJ z+aH(y(^0L?0_?0>^iOwBWLjO79nLFw97W0A1V?dUYhk{rb+DYRZG8JK!KK!%ceU*= zW*$HFZfB|KecoqI`maJx&t(HJ&Hul~zKY@~j|gk;ogGQ?&)VN&hCH!kI+HF0G3=btPz5&^_g`Kmi?>~)I6v%q zi=G~f9|ol}Q(?l97|6WrcSIU=)vWi_*7yHeG>Kv|AYl((Gy}{=SK&JuvqrLJ^kTC zU*7H=AGb`A`^5yo+7DCqydVStbGlIj@B)&VRG=Ck%L}cGlXk8xapX@W& z`9BFhO8)oFmtU6izwa#Nzvp@GtMPx#gx>1Zx3WcA1#TmjN67%_?mAjC)TNcg;9fcj zy!E;eL1!*X{<(^VHj45LpG|_}Cp0TG+c!-NA8TVcrK{^G!Ey8($y86piv^7t#f{}# z8uIf9+|l}|_eygs$m2&BN;5i%LV|FPVuD$YXhLLFG@>Cv*EB-~E1)?!mGjb?-zH=Ir^@>+Gl)~eDNHdxHq2eXND}EN7-N$t2keH|0 zqZMzJ$ z{!cY-7oWz|yO8*!=lz6MkmsQ&38R1?AfQTZg&~~=UE=tU4}gS7dzL#5ewu`f4uX}BJ<`cs@Q(-auW&YP^c}j$aBoh%S@&e|%d#HZLLAY4=AS>Io~`Y;MCRb4oL!2LHfzEi1Y9N z`87b(_`mL#<@>*_?$%QO^DK{{g#dBf)c4?U6eaqGPwa$sy9hrE-n{N#9PYn6Kf3%+ zfA8S!(MkVo@8aTz)ANIV|Mk)F;apXw6h1h**!$-Auzz%Nba}LQEGzGyAMRZqot~_h zlp%45Q}-37_+pQ&Vt@1$)b5YO(mLp9t(hl??FTr=UR9$0v?$=8`v*(I?OF^a&Vje2lbBb0%ss1DBeetxNRRnNX;! zcYcN&<~KTi4w&BPoKD_4S_3moXF+wlK0_j?g;PIiJ%sIDX~TO=arh06aT1WsGhs46 zG=UWx_Y^9iTm^-6PQe_|6B%gf z=u84^ym-6;Qn{B)k)Kpt;Mo0HpBBtLHb=IxN9PxtE2EeuCy&V`h6>MS!SL7az!Mrm zuv1U6Iuq%v^fW_WwLg3&wFUS; zEcCatvw>6P|4RO^?cMD!m;B#zJZ}C^s_1KOg33If(*5IcPDlJN`~4-oA&H9;s9t!z zxBM^kL5Ij`C|cJA2h-#}KMseeAo&Q4q8{X<`9;T}x4f2FK8|Mr`aeD4W9gYf|984u zW%+-5_sb>yf0oBh|38V0kN-b=|K8ljjV+4e``=%G3LMJWmVCq}Da*3WigSL7>?GP_ z$2yXo>?v2S2zG;HN6ki0qgyh^@~Qj({g(HW+`?J_8vW!;k&_v-YBCmy$HD@zURaOn zGx(-O9sg8i95>=REu|WlvVmHo!LpK#D;H_}L&`FKkwT2?m0n!0*y0~uUUAOSB=FA5 zH4@wJPo-0xG_z{gRZ2kGur#NDQi?hH1nO=3a0w|ZnGlzkzZH~fRn<|Aa2e=a!U>B* z>OmlGVdIueygtzO_1G9}NmVuhynHvCv$-!XvAOKj{sC1v*CqvxXSU%6tO-whmC()0 zg8nHAfqpIh`Snx&tR?@aG-Z=46l*$xERp}892{2ie-FOO|M&BO*FBL!>Ble-Dd~t* zbV%ny?ezsqJMufT^Lgb<{pO^}OD34rtK^EYRZ#hiisq7JrxY<$#r@r zXa?wVXAb}oGkWmkNMv&x5Hm#{3Do^v@Tj})l=c1B0zr6PGfG!Q*C%TOVkWK`Y$+_I zNVGqR58QTcY}>ie)$bGxz51S5o}$aJU#dcLm5rH~hOYeoBw{Ip2ov=xAkRihgGg0@ zbWfzD;+E?-*4^5g+*})8T}?h4sY}fLTAEi<$7&cNG`r3d7HG^W8%>$kA@fq{11#7+ zea)g#lSQMLYNRQ#MNKx1rVI@$CwL&Fl5d05oyc-R8gVY<_Y!nR8{L;QGsRV#o zsS?#tA*$hi*;R_G{wZG-%6t5*qyMGyxWa!++g8mcXjT>3xie$o+!eFljPt9`kiZcY8PM`-gRaj)ma;D9 z2Ive)XUrx%VO{tUTpj`*?ZaO%2=kfWxszH|Nfxx2vl!J@3mu zl7I^Ix-*2WeEz!h|BuWw_iv<{ zn51lVDbNs(FQ5Jl_T#sIRT}&yKU>iMl~aK&mH!XQ^#94h*Z7b7`TWb%|84S-{X4ON zte@!+s_I*V89o@Bh7gR(=0lF$>HY z_qUR?wjl~%O6XdXrf3swl~Qd)(JBrc#VyqO`3a9w_A%A&mxV7kO3{HyqA9yM0$V1stXR}jn)d=CE%F*vKVbegQ$pSBWIVnK#h~v)c9^~`qp5>lZ-!$ezayv-nxukJQrz-Ntz%I|1 zPBGJI@18f!$ME=kP9?GY(CP6w?Tu+FdgCC1e{}XqrwhsL6}v^3fsPIy!*>qImSA~W zbqORzID?Qz>}Yvm;jWaw)g>FPkjSOf#FzkEJ=>OhRThp>>FK5lt$EDJr&yP-BuujT zRPWl)?3yM2&T60eESLXW$s~hm#1ppR9Lw~-Cy&eT|NTM#YyQ7``RtH0nx-s?1W7r; zZ6-G}7LjolgisGQrrs5uGT}NqiRTz}otr zM!vH{A~pqXLw4hYO@fciM|$%=_FN*V5x4`#2oi}|Lc$Arga0D*XE7(I(S$qBXf*P8 zBzVXi$ySII#R&C%qIwcJkg!IY2I(vty8-XX_FX)iR^Kf4Y0Bidj&pgz82Mm+L#X6n zTFvvhC*TkG&!ueC+XH8MC6f`Kdl`76fU)}$mY`6fOJge(z2hXnAF!;E{1!;BvpD1nNGebTL3PQS z4YX!M%W{nylxj0qnvw!p4w`>x!odg_A`i zqT`u}(jg>cq9kHBWKMtQ3AskAgX0{lFK`0&wfrlZGO`;6SBzW_+ynP9>3&D958S@n z-z!w3qNxNWVI&ZQMotjPmC!U8hm4F9Mz16kYP^Us0B}yDjE01IpfdsZJLDZFhjMtb zdmQ=Xu{$_$2YV8KfiQ5Xcn*OV?YkG`KsM{=BRr8KIL_aBMrQOHm+kwc+nw;lgFhEM zO9*UO1_;Y{hayr{dKf;j5D#PFBK&-^Bb=-q!pH8w2Jk8N`2p@rq@qL?xY1l95ios<_loa^>3Ii1MWlN-5P5l=~+uxrVY!vzs6 zh2zpN*FK53+Ab#$c~}jPG!~1auQM`q3c}Elrvn72NTiJVvS-s$;J~Bje1@90WC; z*tM+vXs~q9CVY`m%>9>vNHW--?=yeOQgW?QKB6Vas&LVd09O{(LiPv1m7A7F_BcbuZwFV-FE7%Brfj-s z7)stR-INH`wrzAW`b0YEB4(W->6~+5=I=p_>5$KD@B&<6aLpnn#96|}thF_Ht(p>O ztEpMj`A4{<$n+yiJ45n`bfg9+fwZGb_DM%7f1M$D-0%0vXVM9xAPs2vl7)0}!8{)M zqBA5%{e99=ktO-JkL9-$8iZNG-pvvwW<2yeLx>0*5p}VpB(TB?Z^^lCy3z_aUEt>t zz*RDuHv?8Y5GDJ216&>B-`p9J!>0%P1it}C4D;2*{#zauyg^C8VG12~+63Ay0c$Lu zkTcbVU>0*JX$TQaBNB68*nn+wgUUR=1!1SMDKRG8Wuudk&%G;_xB*`kx&lPn{M*<$ zhsaopmqAn#**AbIrPF1-ZpqtgmL!3%glMh1!2l$sQ*txoAnYT@4}>j&$FZiDyCaAx z9y2NBf!sOTlZ)7l@7aqGv!zSd$Q3h6b~(^;8t%2VzXP~pV~Ah6>9VPcE?KgU(dp!8 zo}ML4u&9L~EQ3ouNHt$u3ocbzhwOSZlKZD$Q2ad`t`EO$gfT_u3MD|==p4G{8Qop= zY;HeFldP3ztpb0$f(bg8Ff8$+7Pv0kcaUOac~yu2qI` zOFB`l#n345okm!0!rnT-TSNPilq9x2_quFC%RbhedsA$o7OrjBdrftlrhQaiV77-w z3jvezP^%og8eC`#X%#z8GLiDRwwf#*{chm$meHD9OqwgtmuhQv^YpA9uDjUBVsV_B z;cn&hDx0Y#GNv5r(~z<1OLl{aej<{)x&e5NaM{IgEJYd;*YSyj@8tC5xxny8=IgvN z62x&0T`7+!aT~PDG8{S6zhxYY`S+kU)#kp zK5sP32)-T^1DN6uT@NeY<_s zaIL_5G&}{~a}CZdgeGPFvn7q!aQ)eMkA^Fz>1>INYVhTGEKZq^&-g8H^|mmU+Hk#V zlk0Q1K#=E?cCaK|XW(D3HgE;F;6Ak**(&&weZc3l0j>`nP{+IMBb~>g2W>8uIj99U=yCkhB#a~whQ(-;z(i_p0Y`?2z3P}8i{d?C3?on@= z1wQMkmG`W>-P=)c?78Gzzh{;AGv)Xy#v?{}LgqYSxMUVV`WyGI=w{kV1PWX7?)60x z;|=Za1TOXe%@)HPF0>HJMlN}C{C6p`n+uka{$ti7F$(S8YB+u#f6-Dwo0Cq`gvmKTE5lI!#5}1Kr-6F+U5j97nsnVB6ZRo(W>Ql3nznNM|(CYS~8Gvjta8 zlG&zn|6Z_aN>O>XL@L>%Pg7cGzm@!6PU7Ao_r25dUfzZ5j^b-?Bw6>!%1l4n1GwO) z^*_R9%~csK&?V~y=? z?Q>SeS1I}Js0deOAD7{4E2o0`l)lyRZpoQ-uBh-eu6vz{F_!IgI^w;z2VonkWY_}c5n2~Qaa>K#&# z!kTk0)thUtmi*t6SD?};axXb!9ES^Z9X$Kx#I3C3&hRC&@mZr6lT?!+>dsLRPdN$Z zal)@*T@#+nrPj6ulW1Fxq|2|RoL}$_y3pSOtCzs$;1$6p+mjb5xSq`TP4tqcw2iR> zngIv<#OKk&RO&2DQDJ@~#i3ac6PgkTPs(C)3(;720#}Rw#VwIdyPj;@1z?PPrhEyC zz8IT$T&w_DjUUD;kl8*Nw`4TV#A3|Xw)8Z@RqKbb9Ilj33++b(_wm48-<7Ztu3FcM zfow%tym6M(amC8XNm$`E#`zOTvA`ESQ5y85@k!+m{Y8)7#xT20+TXJ zC4lgOhW1-gFcw{}$W|0%WzltzaMa4LX%e8ZKc+&UPdeHTF!Z5@H*kVe2meL>;E;%W&dyP7c2Z;jo3|dX0 zq{Zid*^%tdm75nvErbh#`C33V2mcA#*<>i#)0b2#4(g30<+4BZeP%2T)>)`uhNLjn zWwgJCyjE|xl|_gzwTchF6<1-h-M4sZ)vO}#_GehgIg5OjEIU`X+D_CV_V)dOMj2qG>msv2GfN z-ANXK=CHT7D!oyQm!PW`>DNJY)%HyCJM+>ScN!s=SBnvM6Zf{cX_&%9aao{bHpX*aR-6+x$UrwHpBoaQ#>ZS4D^3%J!Y59M)OPdG(H8 zv8W_`zfty4+k;J|QX{EsuO09#o32aV(8#A!?%gNBL>pG|5nCtfDIG$93rWn0F4(ey zKPTq8Q4xk31XJvfRB{U%@@R^$v-&dV_FJ8MDMYXIdUtE0-ttprb*`|B zG!FMRlPB>$hAfe%c!js9DqOnr9RiiVDW<_BXnpNzhRbxWZNHW1Z34W{RPn@S{#dt< zswwB)(r zR?~rpg2YKMr^%w#17TaZ>Y`yA;DV;Moxqr*^g|x}CR_zql!Ytl!WBTy+QKjD0BMfD zaLI8P@*6=|0G3l45t<})A=QCTn6D@g_V`TZpb%U=6J3!mLbkV0c(IR26L=Qi1@{yW zbCp061nl5$O}Xv5qDz%+kPKeGJU$~dOJ^)f1Fz9dWy{yWnyRKv9QfT#ugyZk($SLaLDzSJg7*!&KS7XjF>FI6$TULExeHZ#CfAaPn2m!H6=6IR=XQrq&hoE zoUn_O$K=B+^vrCv2&PnAiJrdHtQ&JbuTI>fX-d6WcSa*WWJ%XR*=5%(N_#s#n*@>U zBvN?z>C(vW#t9QCk99AxrxCQ3X9}u`ZI3}3LE6Kp?g6N-{pcSVU9>+4)q>C%eOm>g zJ^J>RjIM&4V44Je4;U%-UJss@sM|6;#TTAhcn0plk-L7VR}EY(fwX0~mIcyo$y*9= zowdZ#mf>0!NW0~mkA`ch^JorNp1-nn^t-uoOA`j`oG`5g1YuUAHR8ZQEL%}q&kA7QWH~lA~1KUF>Zn|RMu~j z0Vp7mq2oNp#rhd!cuAG>*%ey|6!6HHr8kU4wq8XezsD1zvkTt@>CCpN^MuT^Fb(36 z0fPejIvQ=4N~#6qfnlH_ASMrjfuMU8yLSUsyBSTqnbKb52*7cu?~5L0FcUp1Io6y6 zk>Q=UMF{Jl0WqNJ3I>vd(K9E(6qODz0VycNjAx;5@=ckig`6#4Ld%sb>~&DVYu=~| z71-xChHV9MRtFL8g4zF(f+#-_5J`w9H)dWi37EfMJ+{@giL2_%SQYuQa$k2%66 zZ>3&%6A08Z%rA1YZ)SlvE6fxKl4KF6E->ksL3wwVoTd*2p`qX;<^tbnQm)`#kmJ{{ z;ZFoHh(KA6iYI=BhvPcz-O$vVyL|^0DhyxfNVInFeM%Z^@~z!Qsi2MuET*7xMV#49VI^#gT1Jypn(0 zQn)5O%$IpfUh`?-(U9QRIe*khn?jY!lFfVr6=x}2ejwzDdI{%sB=4P<2G>#~O*x_0 zJn&TzeaQ5A!&e99m!MnA9=a4R$ZzmIz^ris0F&QfHeATrttoa95b%zY5WhdMgo zu0kxwEz#$&C= zM?@v$42N9EWe4;6>cl1WDTE)>78B$(8`3{`ly>_O5HB2R;fevpvU zU1gF;#R2gkiS@ePl1&J&RkW69T2=blrfr$l>nS*87|Uu`5zsc}Yc+7!RlqFx3bft= z?FU2bwUrnE)Ie}0k9#}}p-{UG>24N5>X5qmR45$|j|S{O5W&(tCCx!50j*=H&_y_; zy>uS;O3%;jtIFRLv~KxsW~V)>-Z80Gz=;haTb)Qb;VRRg;xHl&B7MFAQQ%V=C^2}8 z7{S$0W)`Cp#waMHOtWt&fv1zXh-|DHWDzic8EZWZV1Lh2dr$EoB+TSgUU8XCi>g)r zx6}SDX#eFb0L^%W+|$+jdps6A%u-eNLqY)6L9jWbmiWj#`Fl|pm8$5Jr%HiUH*XvG zYPIN!8eQ7B`&XlL5E>L|8|8J%^vc@w#jEq5kKer7hriB_FD`z4d;W6&PBiTT+G{K+ z_iswEwbwe;xu`o^g5BMz&&H!|8!NHx7`NqiG*nGSqsOyU` zIc#@tNVb9gm*XXkc>I?#DccMCbJ&%O5d9U5%NE3qY|*-gU(m>G!x2P5`m*c>-DYXF zFx_Hph6|^d6-Kgq+Zdv460@!~+V~we`3tVb)pGacjcvkBaMi2kOSoDN&k8(EH(u)s zmH@2k<*RVCmB3t=pXHF%Yew7gvm9ibo9%KPswDRQ^*CFO(R#z&UzW4w3#*s%@1L{T z5Y?%BCC;Xse=)v>0C*}#VLfNFE@Z8~cVC<>heL9spR3_2ZJfYV-7u zaWKDl^X`mz3G-Q$1~e4z$`4aHT$y0W-&lJJJ%sCuEws7Rzy%Q011ATLs6t0a{^;E?KT55quIng#;IhYH0g zfzuc}V2T~6rprPHZd;B*`=<^rw7=KCovB3Y)!O@PaY`mIUr5iUW_ zd5F--e~g0&D#7Y^Ddj?=AI4NIA=3r_mVbVa9fe6zrgdDX+#7j}l*tZ&nhEww(ONl5 z1A|i{86^ovH)_52hEB^LNEgJZ*!*o`{kvqGNwEpl3uKM3`P2|HAG+w}tL!6YI(Z@_ zRds7lK`i1|%S#BTQ*g%p0H#1$zYFCLugXyDf@)JZQ2P>jQ3}%w60>9;2uwar1-W4% zMrqeOTomAXFW7M$A1Cvs49`;P#tOW;9YNUuZc7cYri^Z7?R)KBAURy{bRjqUJPVuR z8VU~?rXYls5ExFN-TuRGjmU5Pl;E-scpZ34;j%Bt_Hd~R$Sa6pcV zPsR&7*M-tl3YqyHcf!>lT7vXBnT?lQv6Ww@Jidb_?tvAeE*aB5;-oi$xbn7x) zW15Pd{LdZ1)sVNa0j`uz>wMO3$>@6E9&Vbqum-OBh_Obvte`Qs*nXQ>u_9b8ky$lx zwFYNxn?!=&L?NgCOWWzXE;V0MzB?phc{(m|w9%t>ZRe$&sI|k~0#{8+x<7aRxte9v zM+@GPb3@JeWGM#ke%nv$3EnL^$r34$NfNd~WbVeLahoA35xaVbngewui_Ve|^L6y( zKko#t3bCt$t1V#nmW-adPakg@tlI=vC2Y42uC{>PErWHN;Ho5Pt%IvAV0TLbZ0@)& zVBBnHdRugSG+80?hoBN%5?{Nca~3ixn47<2wbT?QC4;~l9?A3DA`f9^ebs!9&Cv-w zPWW63N|_)$W(k!)Y~$`-2Uk>noU`CE3&vN05{Q97^)d{R>fVjPNy7z|Dz9nn!|PA{ zhD_jjDoVozIm^+cUNME2P!kt$OGbnCl&|Z+)m-T8mYiw-M0@(?(aEScXK506YszW0 za0ze5>P=3!Ae+q<@Rmw#KY+U`E&BqLiHSo|U=NJiuSrIr% zm=psG?MH(xDs`)rXpzqn*2aQzFB>dyAiH%F)z9XX|GBIbduh<6oT>)QwuCabW!O$v zN-47xOiQ$Ffv7IOOd~|?8D^G3q%1}~_5a=CEG=Nxr=4j4b6Mh_UsP5IkG-(2} zJ_}6)n9K6eECq9!;M)jhZ3%(~Fmrpvx)5qGkHZ_fXe-uMcd486?cb_$oGiNTUeg3Q zS_$$^z+A@K>cMPGzu%Z~zY6WYND}@cZi;VV#Y||GlV%a)YQNRDaDxlcmUXBd%vDkl z)q}agNh>DcBkf?8{)`J5`TkHa+X{2HVe=<0QldJp2%rITnNU-Q%_QKkc{Rn?+Q8KP zhzC_(*8=n5!wt@+Dp6lsk*Zm=Sx1^FgSqTYq6N&jCgWiRn|5*dmU=DAz+9%wEdjGi ztSfgWN%0M7ys%HenqWTbKWlv%XaO^&$&{sM1z_8;sli=y`!@rV#_?IgQ||G)0Pkbp zN6&Vv8JPquG)C#Q&nP%~ENM~&Qwf?Ebw%fH^C}0~*o0wfV}dr-DolGc49C>F0vFkK zFgLjAZTUiP3yDzb&S`C?&?Tta=nA!IgtsIzN?-{{l^KEvr4U{*7OxEEGEK1t%!s8o zJh?)t)rG=Tsyz27r<;`gY&QcJSty7XL}Q*sK1mZgnFJn6XFLkgVOKxMSfwV!hIF$A z%vTD^6~N?KT7)y@OQ76Pb8X8PiGDPu9GI#BRwLr-dRHrM`Uav$TkY|CE;n8TtgZO? zJ=YsAk?@x4GU?b!tBO~|9N6VaX4ZqW_WIPo59x+peFS~3%pGsZOI6qs zyq<^`1}23!VA$hPgr#2BlZ01-zlKWH;s2wg*Fl=cGvycSmK@s;*I-oEzhc$`>>J91 zBfqwygpiaf3$LPDo=K4+#@7zbgInwS-jRYT(0EhR9yK|=YM$J~FYB2xfA*DMk6hQcJU19$M4_?!uOg-i*8 zYN;}UX0`I2y>*QyC(rY^(DT#wJ8Ny2%0nI++nbiUb*m2A8xhP~c5yV!kCX!~df zdcXvi^6p%UYqZT0NR!2_#K#0`D1p?J2oqBlW@Hkp0pK&xA6|ux8q*B`q z&|+5>bDX@$jpdl9>xQ92YOxfV9HLn6snSYiqi`KZ10zpsU8yxWn8%ZZiI}6mZE?0? zzI~P65=&j8N{(PDD8L&)bq4t9UDKo&24jt;p50DWv>vKgKT+E#Vbp&cg^MB9H@2qC zU1pZfT52lyShdVV(i1b9Fu$pB82}p(jVLG-&mWpUyV`yawdkAIvNC{h=oBo*Gd$&F zqzi+MTj6)hJn;?aLGk#+{*{L^E^H|3}pGI!F!Zi{KIYI8hh2IhKyWr7~+&Y@7 ze{ymP-?74VIzz|$#dTfluN@+JM95MG!$Bf0@P04#u!l~AG-RD2>0HQl8wZ`vzTGGb zeA$GKJ%4cU_^|k+;vJnK8JMp>!@s;tqyC-1q{bjX5!yWb+TJNYNI>*i*uO(v z&ip4ewj9O2A-ysXu0#zq&Rt}>BBh#+OYqLwWJt8pUo#?)&+Yt32decnV(FHcy zz31QLDGr9#vp@V8@~a>Rt^1EG@UFTT{yi*>KRDl#4PNCQ9w|_e=w|~vA0tL87!|Kig$JK~S=#4uK(pfgn1WP;~$-56|lz;c~ zXU^W7ij*Zbc1zo(*d_V?_~O;kVd2WZw6kU@DOXi}qABDM2T#d~#rC@N3;9YIQJnJ|Q$p z^T|NqL3|k`IF9<_`0NyO4k=$jl}1pa5r*bcj)n*zZ^q_yNWRtQY{9c+3WMNY_h|1^ zobcb7mv-f8{@fcUH1cLl^giLgx`F?>2jllX$^V1D723a3B64vEc{m)?mSh4Qhb43m zdk>Z4aFA&KU?@EdI9ZT4?=RlTLz%ERq#hcNzd86p?Y=#Wim1S=u}=)F@_W1@FtCE6 zxIcc740?W$9OPHW(We9tQXpDUCMW*p;0HN%jxTjF5mj=EZw`KtlgR(m(-_3T4b{a% zikS!XeW9oCH*KQ zh*3yRSt1C}FnX6qO2BrgG9%DDv_FVi60^Dxql~B7>X6R3U^YZj__#@eG-Z*43=wlx zf^Ot_Oo&1}kV{T5DTls%rJORNJLo; z6`QN1#T}B)pyTMluAYgKxG^NRU5AhluIv70cvqO5(Xm0 zdt&NI^29n;`|`|Y)0yI+>VPo6?*2H9_lGRKKjoBw^%uy&3#37{ZVy@E_DI=4#n8Fr zy-=9}1S9g~;9UETPKu>{MeGbvD)WNAQ?BEvWHXYTOXWpwE2dPDgWGAYg2+6Hf%1?B zb7tLOxC(H!{e57?OR|h9Rdb}WSN)!ehAvi88mZx_R59O*YY({3ckZ{wi1?i=u02=n zrwhW~>dDchdUj!Li*-~Wf3Qud&@P624`iY-^QfdVHo`QwRY#S?38cG+3rE7>D$g{5 zoLA>^0Oy_@4vAtYTS{X|T#|sM`I$AZD=8;)8i9kGJYW{7$zy-0?@QH%i*%aELzyQ| z0}6Om4mldO(uRYFzWLtTtGSz^D}r0vHuc0(#y~veIkbB;l=xyB^R$G#?R=32lCs8rseN7 zedP+qR4@q~;@Q@xg$U=|msnS2836J4&z^dI=s_mTZ7lXbvnw(R3G-UobH^$*-hkQHcJp4|X z#^n~mtzv$V7Vf=O7$P@w$W{`$dQ@PiO;^;unLdqEsk+HQKPV)iYu~U3zI@t^!R5&m zDP15^d^IeRyd7m8vhHP9rQ?#EW3ONbvag7J>)>uVb@!^~TJ4Z;@qX^>CoTWGg2O`I z$0g^gg=1gH9}^nSf|RulXkkm<(b7oz;!al>2{|s2H^Y^Z+{cAfC{fcUyBDaE^Mrh# z@Eeim*MEPySD6%t8saGVoPpHTcrqn{@2d23xj~aB(^H=eq<#hZtj-Roy2H=8xyr2I zJ4u7emW)C+wP*cj{c0~wAq<(>mosO=OlZhYOJ8{n%Kd=ewe(SW?hAbkxpGRk4F$%Y zqD^E>Mc_f%Y}F9HIL^{p&5U_t!)nXD-q*uw_0(7>_dKg&rxs$69_{p3Ge&)uXbvKO zKFNQrj5epql@dDPoY+_94gCn3O$^6!s=5Z^rER!{vF0@X4t=Zy+LUy{8jnZU3~KD8 z9L=v_1W2{&BEJImVHtc&tVZ7~Lk>CvchJdyPeTFwqhFtjU;irCe|*A{w9vrX3xJA` zf0vg~W!5dOfkj@jh(Vmi&P^OO4TYYnJUs#GEHpzpqZte5I@olCMmd@m zvO zFa=oY+F0potk{y2*)97OwXRneX$T4{BBb6G{GA2<&}=z2*&!@QXG($K$@DTX?>IYz z=U*$8Myj4~lPQv!KwRoqy`0(#7WtP6n+K6E6&3Tn3OM!_Loz^it9aU_+>P*>UCx;h zbjoZ|c?Tj>Mi*JEYPVe|7cofwVTwu_$1$NAB7tMno2g-Hjva`13S{g)G*3%j5K9_8 zBoF`lzyDjSMv$y&j-)? zHP6=_2enK=7ZJ^U+uFE8x#;qjwOQ}xi3l{|9xs;G8s6K^)?KXhysNPsh;pIky+551 zyk@hZhcbfilNrBZ*HR?cbrgm2D+q%$4Hbd)_*|9a^U1DY3>~QUT*rYOH3tD1SEgdW z4gFJB6Y`Pm1UzCN)4aH@U}-mtbNb!YzjO@};jO4!pzweWP9`i>G3U5y#ahr&QA?m!iO7A- zMsC{MW5R3U?slv4*){|{GDg8jo|VDTmxT^c|!hv{N}Za z;j@^iS{~39d!z60fdU3?dYphCs}B?KwYzAw>tQ9+T-xuJHD>uih|G}IFV-{1JFZ@r z3bjA2Q;$v{=?y0$WwC4}@Ay7g(IbY}MU7Cn-s;$BQj(t%IjW;Bdz6=cE2fIjolS8E ztFtW!=?bca-b;+RZTp8ab{n?QP;lF`vBBP3v%6~WRW})%L$1W!cw5pmO%%s-_5^8^2J&QRqo^gxy^(YYdLJw?w4AN;iSR;x_* zHMDtmeNqV|v7TD)k%UdNkS5^*pg_+NO>ahe{*k`L%F(ryTvd~I^(#2Ax=8tfo=p=v z&QBNkwJC90ax+LK5kM+OjWC*$aTbI=J_%``VbEYCS#@m(H+g3)_cqar_Tur+AE55u z@l8)D&s56je1M0s3mcakJ(Zu9;VjU!mp4+q4_5DeQ0@JA_1=$*y$iymK2OafD3{CT zUgaUpL0oUlVmF8Adw|!#8moHURm(GI*>d5LP+dr=T`Q;Nw}g2txK@&T&K!V7DA}*8 z+gL5Cx|sfyl@I;dA-}TmjPon<{`|E{wl@h>vZzv9bn@=hj9{n=6CRBj@p%RzEt$Oa z;{;4GbV0|^=?z|k^+e?SV>;9Aw%1e1^a!C2_fVKd>}^4csyP~^l-Ab=YgLO~-? zY#AhN0AOqk$i;)&f%z2_)kY;GDCr?Amy!K1AtTo=z*P%C%qaz5C>=)P7DP`;szDZo zmFYh64|pNchOrFl{qLi1HWV`A5i7(7AlKs8ERiZ0@MiiEAqt6E7K`#iewXYYP2gas6rOL9XD*YOc zJ{)N$M~I?izDr(7J_e|V<+I2qS&R?)1>|UjyAc0SxNu5WHL2H?!vA9_p}M_8LJ;ynX2v2-Wx?JB;^NIKCph(02#fPd;3Hn-Rgh8dJhIm`(%Q36Eo_Lt#}JDDwd+3GD@P4oJrgEc$Yz-wG7|8O-f#z^gfaonoh zqMrax3m6$TMyq|GSvqqeBT(R7a{1S9`SB$$^!dfO)XPAA_2BYSuJ==qrCv`>*^@ml z3*+=mpU?RK3i`DN!4{vj6Gfr?5Cry^`WqpKT6XCS)Ddgt-EMbHuOLK|SHvhZw>hm99HK&mmWhu(A##vE{|jE>fP*DI=#*%9087 z7}-6!INdW>y6*B*a&mD>{6vcT1Y#i)mL>ta&d-pZ>cjCrvxJequ?6`BOvGnuT#Hgb zqzDiSXslP;g(vi`CxR|cWe7r-%65AzZ&`6b5Mo#b&q)^Pnd^`Ws+0({!h#T$hITy> zxRR^~+&rpO1|dqgI8M_f7-uQ7lpq{Uqv%92uld5PTx&ZK_t{xC4g)d6MW3_Du%yb- ztLv_pFdhB}qO;s-ouGi6h1s-w8mS#FiJDB;_WO{qIZs(3VM`EAYxl1`|6p$%^0AIA zIDd8g^35xE?(Za7)D5C8W>d5_wUAArDg&5PURqTlS%KqxzsUWA>hinz%47r4&a@mN zj$#$${Q!RiVS<_q=}YQzf@q-+f&3|$&dBjK4MM7-vn5ZcA)gpnkR zEGd#uGLz#_dmQ}^WE)NqQx4jYs2@szbw;4ENjX7A`bQPpjhG$DnLsKVSz_79QHH2v zL8RV;W?MO_uKp?*R5+9**J6qdqv2|9=)wW5L01T!Q}pT8qI3oNloLJ%Utk)B(B$+? zsMd~lTPAgn~6TloE11 zP`PK!fBXqQ7=Dy=b2E4k=#r?3YOW9PTs}v?-|rtC9>V|k`~C9&_xt^W{}>!TeR4Dy z93DP7_>ca<;X!}!AEbXD2*^ZAlmF;%Jy&^gf056p&tDpXub(gZX}@ZK!&eHIy0OI?$zY|OgG zuk_(}FUY_>ICA^lKKYgm+{Xj?&xf^Lxfqf#h_a8?uj)w3pFPP@CC48U^=*AS>pd68 zw$5``EJVuY^&Jx|8q!4pUSEbd6SFwsAIsV{jpO{!zB_OayQ<59<1^tU0X$Dig9Z%^(bEZgHt6MZ?YtgzUk{$hg`LR)mtm z0M-ehTnaD`knSk5cuI0X1gP}KI zH<$;JFM0w7K_5no9KZ`!68Lg2cbpx_KLoEUs!q5?wRBY{aJM(s570rq8S}1Cp=BgM zDybsW?bLiq*?cHW4z)QtZ#p}KMhoLAQeLmnrZsr#x?&5V6U@$OjBvk=~$6bM1thKwt>ZqLd}fC%b-z3ivfk zLW)&;)$$VX#oXIh>%jhcZk;P6P>)GCOIZ%ajCeCHSX5wl=HLv; zhu>hXd~1iKBPEQ3LC4wAsdHe7U~9zcNoPDshh)${cygeAP)t>~BkJ&*KP&es{~?I* z5$ED6kfJ~^SybC(f)#6;+o4kWhF0pK!Ligg*3N&HHHYC{^ z%#@09iY0>T-WzQarn{BmzzXSpNtI2kzY8F^iQG=2p>tWN7b`Pz9DGbS~UWtPEz>W>FNg6ioGoha51bu!5HN7Uy!_N(O19ZsiO1nJf<>}1p?{sK1?Qp$YUe`XxYb@c@QCp zFZy|1JWXuyK&ZoGWO59e6G|SJ^YzF}Bf0cPgWQcM5^ z7e$)?V(un)h}zTVkB^?ih^G^hMG*Or8qY^778hdk&R-iJ4iud9*6N3BE!1^CeKOSI z^XD!!c0YZRgeex(W1l})GgAydeZqDUu+N|IDjDKE=RR83H4j(Flxz(Nm~qtLS(;~; zQ}%RB29*PzFp&z|i6&(98a6n>n0|ealtcvh5tvj_Zo+;+UX-|+@C359N@)S&ghXl) zDtN;`J(JxB2T$F;+jj>ztajHiuM!4AHHJdIP3r7O=uEKJ#(luZcO?-K{nsJz0%*ND z2iE~5KfQZ*cHtO4g5M1AKbfgesgdBU%H9cS*xoFi$)?&#nisYicg|uPq}1V!ask$k zm*z;0Rt#;l9DR~iFvXHBK*Lc7A(3}qszaoM*wu*fs2dR zrVtgXg?Q(h#coBtjFT&mkd7lWiI^mTG+AH~QNEK+W0_G&FF)gfBrgSm^wK> zi&`{9KaaAs)>SOsvt+c&mP^5?)+9qnGNzuM-13eq;a07rq}C`_LsW$nZb9u>gsU$4 z(Jv%0w`ViHzJ_0xZiEqyuAT{Ue0Hj)dMo$Mh~*PfyK|VqyzomK=fpr8{eGVVj8Rik z^LOz46e_bwtZG+pz0Etsx(BLc0}Vs);KBTzAcX2wTbB38K|h}hTgE5Es&@i_5TXl> z=qbT=sr*Jws$KJ(kW1XwhNW~`c|*v5_xOBHBY#LfbV{8&fC!Jr3rPpCxO?-=I-z4X z48{pf7Ts|s7Gro>P$0nb%)l`UiwP`-umzP`vTP4Fl07D?RHsJXbL9(UP7w7HE<}!h ztrk|#s;-V4VU>}B;ZLh6LQ0)x!@vv0p7e}E%1fmTrkM(-LRLVD7xK(EZgyl6BkM<~sj zQIK1-;oVwWKJY_f1p}zESURVoKt2y6OD+$Sfp8@Gp9sdP{6Q8^6HMJ`8HS}ym&h-% z4740538u5OW;z!lGBrfC261&ip!tyW9S8KcC2D&*OCT)_s_!LcptSG&m>7z!vuC01 zuF_boM7&h98uA5~N4uv{A{E3?tI75YDR>Pawd1jDag`D@o29r4>$%C+q#BUL?B83OytFW6+lymUx9Kl2L(fc#op zC5EK)>K_>mJ4I|YfT1nhg+0dCMyQ9CFw*&U;0_ME{SI+YXiRmWtoyh8wa)!9NcFRx z9B0f@T8iJweSQF}P%voO+Ngflf5cbH;*ucR|8F(#hxx)Dm}hG;d_nxcOU+O4u9`QH zzw2XK_a@nlD>H;^y}UnY#+!{afbmk!J!i=jo(M5a)B-#oUpa2H-H^rs)!OWVD|oh% zYdk`XsR^-4NQo#^-Zxb$($ChWrfPE~Ko&BPjHz&V=#; z18?!dEadZN`*~IVto42KM0-pAk?|CoYTaUJ1xKy|k$%YNls0Qur+FovcRG??eI;~i z91T$ZE+dtsqFKUBprQuy7f1+3N5v9yURBH)x?y#iy}deSQ0a(mEW+PuZmaiC@4VGd zJzjky+;4;{uD#(meuwM^D)Ow1OE_2^;_j82l#ebpNb6QdiifF0dGMXE@q{7P5r{JD zGxG5_qH9o}Z*=v51;V8J?Nra--pa=wff5A^3>ct)UJ=f&>f2QY_byt0?-Tmay0wsg zCywD-=7}B}cEQE{tZ!H#2rj8I8fe#gdsoiKKnAx$(A_>G(6P@?*i(sks=-b?}{5+hGP<`2H9;K$O4Sch0xpBnSUe+pWNoEsvml0Z*SV%`h={bWl z240!N7wGq-Tjb79)-sw70jY9iyxDr|3i5>%1zgg}k3)WPtg~2@jO(U)aMyD^P#W8* zuP|fe^p){ugH=>8&Dd1{tMhg!yaabhN5wUER_cXjJ$u zZT74%Q2nkUKUi&qHFmCmn9cUip1QhwsmTu9x)KjXXffMf)w3b?uha-?( zIPguw?A4&Yh!zX%G!*Du)CRfTs8i*gom%F*Y%%$udOa(ST(>^^))yIbo!^sN@)2s~ zGo`kjttP5*PP)_8pa;;Z>%rEYX1Qc*&bG?p87(nY-HwiLmV{nwbz za$_<_7fEs>9y?cTu?>XQlwx!(A*yN?f?M`N34 zc{_1iti1&7qbgCD3Q8qp2A*)GjwzA33j%{i3ImhfF0P(`T(^6%HZQJGC~F4`BCXO5=bQ(g3*wMKU#%l6kgtBA_d%F z(tOMykk==e2hot^b%p2jt$9$b8Z}UTp zaJsYoRD01S55cDSGDsfkFW?B$boLPP`Gx$3`9G$jr^H0-^5&U zozUFGBk42{7Z)Bje>3A8N#Pk+!a z=l?rAeEe1ayN}P8(f?>13n^PS^s*9yS_2@Hw4(8Cig13;SQN-&8z?23owM3 zYsNleoyj5TC|4oFW!FNT9sN+la;#1!hNIwFhwMcM6<{W3ntHR>n=bVTht&8~Ls}ap zKN}F+)3crD)N;gZw7Y}yN&fos`-!I>vzuqg1Uvt`(*{eu~^Fh_mUF!RZ9QW<{ zyS+A$Wgn18yOJfmsEVF?eme zx_NPwmbJFpL2B^X+P?mg+lEWG=!&Z#?p{Nijlo-Hrm-gmlS@}>6fZW&bg?mrC4Is` z6XH)VumXk%LEOjFZ~fDaA(?8i-xXUd>DIks^<6!>!lYe~c4f1|2o38leZ`_{`>G=q zRQKGX9}q?HL&E393VD!Us!;2}`W~~Ch?J(8aM7Hx6;(jkBq=BZHY9a~6ukZ0S8oQ@ zbgMdYn@2D@iP{6|H;WSJTTY%%pc(T& zsf^-5`QQhiPF|hAyZqtw^{dODkKerdTx|$Je-A$W?bY8~n<2e~5sxclTCMZnF5bRB zKY8`y!R{o8e08(|<{s&S7tMq~0&LQ$LP>6kkW{8C)rJ|PKIukeuxHI<;9X~$(ju47 zVy|_r^;B@wG?z75%0x=K|14}8PWqYr=YOhw_<;s0-V?wuHoXos`52^y8s7PkC44oH zc*Pbg4Om4@3zW1L&dMP>;W1=-Fs(5c%!Bewa$xJrcgN>HzItb$Sud{XROvx(Z>uZx zW^?Y7!^6YY_Kl!x^eMlv6FVQGcAA0=smtx=KgfY<rqS{=CjP;CjD%Ij4J zTA5_64&BM|U1HB<8mk4ebSkPquBAnkHf6IhYgCLXMmHR~PXAb06vlV08UgJNUKBMb zclNG0IH)zWtt(e|ktMf@*X0JawfpW)axc6pt`4|<612A$tId9y=f4#YqMtQY`X5qM z7gl{5{QCJf_-y$8i$rlH@86~VzmE?O56k|)gZ|h0PxtcK?ETl~VVH#swE4EE)BAA$ ztpu?zdqXV7$^Z48e`})nzm(JKo$tjT?($lJ_%1wNYtZl~aCohO>aM*7mu%;s$$9e2 zV`F94N&Cx9d?$_5p3+U0`%vDATVZZ0xD&U+hT*nxE362_|0c5WHuB%96#;D}de3k$1=krD6Kkfck@#wodl6?oBdP`_FX>ioTG*5=4ukAJ0JnbDg%{32R^qC{h z5))U;TW|eK?;X3qsROXo0j{RqeY?LQ2uq#cs?C-Fx`yXktrYrKbzCdH#&7fLjA&Wn zwPrV3-)Aj0nZCC3SgSo#>-!g!`>o(zuuS>*OA&1&X!Attb?S7**15ve?aeNBR<6y0 zhn>psSJO=Mi-f1#ZIhlcTTl-+g?xi~lI5p{s_0RMOPdii7#tb3L9g4j#U}GM&*0C08IfOVTm^0c58k zV`L-!$!m9#PoI{LXC-;8kwbsyV-P8ad8HH}asInJ5t$-(c7twJe6>+?Q59T`0kYNv z*?_n^4;#^@qX(u&>F@>VbVw;*nLIqFf^>rDnudXu4^-iHkF49n)f1{zFEuCKG-(Ng zDCJW!JZ4wtJw1UCW>Z_cagGIFVrwG$(6eM&3-c{s?6{FTKF8-S{up{pdcBsn$s z*|Uzz%EHIj&lmr!C;tImq$%sF;_h89@t0cxm&kvE{!v-}domb&&Hr>SpB=@drYYGC zSwwb2&abkVblRQ=JEUX1t92y%=ydk>NFQXx9nc;hKpo9OeGU8plAhv^ohHezSCsiW zXu`J!QA&29n#S?jsp2leY8gz3%2uc*-A(yL3U$X-^6MS)|0}^&Xd;=YHJ;AVR`-`m zk$kyN9>s8HEUf@Z&Mi#2%#-t%SRxUMH%3#JoIy@>qAnsi%Zt*! z)kSHc)0j=Qqz6?+{0%>`X7r}^MI}CfB&dorT%8ml*a}!9J$HYif^>@S{=LeJKc=6B5hpYVN$1^+DaqHA0P{H$FxiY=V{Q1#Fy#2` z)L2msq17d-&o0hX#bZ;eQjyp=;T~hYq|Tip;cWT$m)`$X?!P-@;XIf|JYgHnvF!do zd0fu_*?;`>;OqUrkIxP{qiM<#tZavyj7EiV7KBihHKyJboigD%s&cQuB71^}84E)q z?>E|Yf@r!=5*E@lxMtuaZGBH8-`OD%n_@}b-58Q6F&}-S|Hqz-b)|U(JtasaW>A=q zxsLnt;_@Qp33GPH36xDCznokUKS+e*PJ^@u|3&Z}cl^(!2mjSy%%(m0AN{Aej(Yh( zW9mtY2M*cJBUjwS&LekBubfA2I+y?ENicOD{U2wC{6dp}XM&u*d?g$=PWbQ4OC2}x z8SP>7g#Yfi*TUmI>wWRdy~1ae`+xrG_~n~d?%dySj`sV1a5Q*)P`v+xr%w*P^8b7J z>{N<-JI=AL399)U6*vnlRRz&bqNAcs)(| zdgY<^z+9IwnzAZvuv(8sCK_bMf&?7&9tskvlHhWY_%01~Nz+*c4=y<-A8ORP+GU{j z+>1@~t}X()S9(7oC+8#(goc7^yJudU+*ly(#2`{g(#A)KNo3TU$z93ojSEvO+f=}I z9p}B&R0)kx`HVbiHuxb|8=0j3z#dn6D4nUOPN+ zG+}IjMk?(R6!tOZaE)>x1_w{wzT0;P6)3KKNhWmgt^iP&FIgq? zQed!Tq+^;!E1W!=fBn*3$7=Q!`IgDWqhqB{vwciPWM9yLU7BCoAF zIv=cyrb@u4i(V^LZ0)f}lq}&o&dv^bpI5}^HvX>%?!lA)={OfDO;Y)D5LJHOCm+tZ zPqa!y&Qy3o4pcF-jL$vM^LXU3I2AqZ@70UB-_?I~mEyIBO4FFJh$L*zubGcWM54rk zKg{P@?yU33DU&NhVl=7BcoR zW?qV?CT7WmCv&7hU8t#~ASlx!Ai<+XRdodBL3u*KTsM*`Yao?Kf#`@6b}c2taA6vz zNybL5^Z)+e|Bv$>dGzRKp0eSiN6vTTw4gky+OtwVUYKDpy$oKm8!8ADf_nj+Oym{O zLMo!UuTRa+1QD6gYl%+DV*#3PxcFiJ*Q8~e!Mi~vm1@S^gBPlSLMefOS6r@=TxH?w%_# zWpJg$CqbqxB_hqdD`PhR$o5H$Fnc@_P>p=X47HKh5~D|NfVyd9z_^7KtAtUK-^M(W z#}7zD+gVOITAlY1E9wG-p*lM?#|tw=B1sHVdFW>}oaDD(%U`G<1|am9_d^X5@_1u> zHYCE-I)fJSoRo>!v9GwsHdKu2L3(>yMi?_mIDBK|Y1|13*jyfGpQSVi3tMAeZSy@h zn`54xP2u=F6gXlxq<)ns0>^QZk*nq66Nps;OCrJ!>Bf>T zLP;`~N|LEKT3NzX>yUt;{Hiw-tBnHyRsr@M`ImmJaBj(&o`BqvmrQs`00HOZmc05% z=W)o$t#eDd@9C&#cJx%DVm zCOgW-K$O|h_f!-lUP+T~C_ct3Ap}L4kt#|3t;!|#-SAs$rtfk|MnZJ(`04WzE~!%U zTmxl(085NweE9{vC8JJfq)`1K*7Ky>ZCLy*ZWS zqW6+}83YzdhD{p5L#EX5KQ%)okKM8>?Z38}Z>2ddDbKeEzMtY~xv^ zZ-?^oa9q!ervAT3K>-lH+F1wP4j(EMOo7fHn-s+HOIr+9Yl1cefs zoao-nSi-EEh?{#@X$W@!;MO%He&4~GWdSwR^ni3W* z3hn4xr(WzT=*vWPAy(B#OgO}RLW{yR^YPVd6X6_ zdR247ffPD<955->hYMvf^Oz7cS*YxJki=5z;SqVIQr_ud zK|35TY~j^41}Y$2Fe)^YdR~JRsY<5mNy(X@%6pwQEx&oNTx;F1bLp)Mt|* zk}`YVxK64`81?b394>)=+fZ2|ji7C*?OPy1wr?T?@@A{q+**v4iXBK~{W@dt;9`*_ zZ3S7`2GBsJ>S_Q3wK89oF{z+0*~t`LCS&m1*ZZeP2+%V{;*14K4MXcE(yk48P*>eE zJMfWb_;`w*aN0{YFlL)lB7eCzYZM=$3BzE9TQd6aTM2p9x?#2z$pRl}w`GUgo#GpS zb);nLEb>_rf|bTL9G!8$WVlCMfY3Ro5uGC0wx`%@+VGmPWyY4C&BqL$KEXaKA+lhe zQL#?~bMnAP5gCl#tzz{mjJQWO6Qz zxWZQM29sQj+{=+;;gpvrO3m{F^YMDwhv2FvT|)7V=7P}t)+%Nux1nFoMB)1x2n2HIkj7x}Rm;L$ zgnRo6e(Y~v+~PP4&;Y^{pCzt3B#!gwT@Vl9B1?fyinT~LtAgvdQnb3gVvAc8hrdI4 z*aP<^48gt$Lf@l_4?9S`lqKSkOC0BdvE1V|cjmqWpVxe_J=_X>zz5F(aDm9rlO-3q ze+B3Fub(#m|7^^T{rQDAV}2I0q;~>dJ1gdVv_H$@f1W-q`Trjbo(#VF|KH08QjMr= zcMc&QO=-xd+G{M}y%~_#A?ZLek=iEUZgZ^@dp!K^1sS*pM{d8{C*P8R`*N_gr-BWm@f7rnJscqX%JN4AI zrnYUno!YiFwQbwR)OI^{J9V@9zwcgqwuie1yE)BKu3Sl;{O-^5y)oQyTs%}o$YkJT zEeTKk|7{2Qtf}cC6AF6t;2j{)5A(c)YK9L&02eJ`;2IB5;zp*xM(~GFh#-4>b#g&p zUS!Tx?EXCF;s3!e#YeB$cZY`Er&-N+P_q}05VZIMIggn5f8`g8zHxg`|H0IAzu4CR z!qzV6uQK zpK=3ZftE)VwVAKD6+QV~u=i*OOP11cRw37R`yxKRS1v}$e>#tth=>R|E>Dnzkk8+5 z{<#~jBt_tQ$>L{sQ5nWR+xe-}4KMz)USd9{oG1_K&O>_q>@a#C8-Q^xzj8&QSEsc| zf-@arZWSO+bn!f;$vypzhxfVe7Z>mBEY6}(PA;P>YQVeS6Kcl2z8hM`okboii>teo zM(vM@LRcKcJkXs=*F`Q~_RB>grYI-7LT15f?@XI-CRXe}@{ZI?9B_DSLwj8Qau ztfuj)J|;f@_jIbH+c1)ztOR^#rG&s^@9k{5WQgcuhWdx9@N@o}g^Z(iN$jUAwGun&u>n3DlJr&@|giemC z`vi{dv~jlCQAYdqzF@hKSx{ZTlhREk4Mn-gN~&}Ajyb6&HTksaGGTy+QL2*$7z#~n zjhD?nKYXA2-odf`0M5K8T3BML@sXf;#ODoqym4%g_cyDlbq8@6n4QlXLUtEaoBBvS z{a`6Nn9$Hoa!sRDWQ3YaOo(Vr8x@FYJfNl}Prz;-=rsQ3?=sp|WvND}?h4Zfn-xtJ zvJxSlyi$>%17i<~Wmt9Gl@$?z#{8naj%|8T?h|Fes&nKf!4Sv&ii?9EXr#k0m>&FT zkX_&I$=)L7#}E5Vl^Txy38dwNjv{z&EiKQM3 z^^@~#g|(MyWz)q|+@5A@8+wAN`P0#646lVQLC6|;M)6?1_^cHwO9L*c3EjMCf>@6A zbMytc^_FE?SPAMujFcdnv|;Sy)S=mep9n1u+sXkKj6$>}RbT7yFXZuDaF%ov$zjWp z3P0zkjF$n@c`Dq_C1?8I_Ofl?7$sPv6mhtj$2ZIMzpY$s!%x0Sj?UMNeZz(<;f@}$ zAL!ZJ7FM5j8&GSPbM4QF8i5xxdosrj!)u2=j1t0e>bdpevI0-qXE*tXjS-K^_R6Sw z(h=YLsic@@fvYw68y!U=BMW^tI!7hMzM{`9Ck5;e2`TtQqS#03VZywRv$h-xw@U^4!1*XDqWDbthsOj3JFo+pwm%`W-#>SN(C$HWNrbK zy~)s|;npqL*{+!p-u*z0n>!eqQQ90giJ=4b9A9k5s5r&<$nVhw89wQ3oPCgULEpf( z*nUaU0hDtimXdA|S9v7U#`Z-T<7Xng*`}qa7*L{PSYrJvusN;N5C=~gxqfdz+%<_f zWZVjA1Wj?Vp-&oeCwCC;(Zfn*m^Mk}!Oiv{ctd>mMDRum&NoYZ7I^SGdJeCLLlC}l zRu@riwRB1oZ%-g;BJh@v?x{n43gcls^AdI?Dkifie`(B)3U|tQ7{>m)9I+U!qV8OZ z)FFfk{i}iliaE{uHYSS$Nyt7i%)`Vkm6Vl;jVAcEeKJ38VNuq})Ng9Yl$7x99YuxJ z3EQ+dWFgFmX$k9N%9Lqc4=tXi#EhpG{>r=we|Y3I-~?jPg>Zg1t^A6$1ikVmmffJd zgO*i^P$vydWc$~a87v$QiJN4&3d)DWB^(Q$1Eh60FQY;gt z{dR@*2oNu*j0~$5kX?~aYkv)M+vY_geVES%y0?xS9I{W^<975fH~X?|>q+`m0b+8@ zV3dkuOZQNY(@`OR7sNE4-cqNtPuLHynt!X1I!rdzB|s@TPmWX}H+_gvauKs=n;Kw8 z+ftWX4934N*UdI!)m?#@0t!7I8_X|6gdGR8->`bgy{<0Ovyfh&*ENwkRjpZriLUOj z|HJKxrCDb`z3+I}1L<;4{x)^5fpXjvAU9KqPJFzJ^i+SHww>!VN&poCFTmDx*Dcj= zTz4@D-`qLEV$x2p;-^uHqhd=sbRA~qhHuhUyRzs@*y*Ph?~YtCSWN!QmWp=FfVij1 zFck;b>S1($tT;`vn@|I%%N}%|{L#h*o!1_0IZJ4mzY6G_)tSahDM+i59`*3`PG>W>5Z8* zLn=bRs9|q?G1x>@mX?To0BcENjaZ+pBiWPjP!1UZXW6-{ZMtv~MJo$#-{y|`b^exD zQNE-8ub~fzjJevOfgg!BjB;vptQ!>^Rw>}Y&?RhNp~DJyjXR3!dG ze4*Yw3-aJgaHKIwt{11Xlcw=PYE{s|U~k(|j+2%7dCcuTmY=_*&M*#_^?OY96HaZG zBW-LnjbH_=^x5a__s{i=n#A4K!u!=*2>;TLTQ8sVu+>^0^IWZlw{TggNIeGVsdC+m z7SxqwmBCOU+HvVBJ3hv>>%wsrv6px9CH&6o3*{}2l^pzaXp_U2&GNXH#Tviee0dI^ zShD|UccbZna*39em-|l%U?;z~X*X3v!SvUxdU29#O#HQp)#s+C2=7+ziaIshNl*~o{gKdJ; z)R#Fw@}})(Ew$cI(*V;xbvt#G@!P&{5PIbjRXuji<*7k8-r(=R zRK?bh&Z?ohwDqu+*Tln3Mw~GxMQMT1laDooXYHv6<>W{IR7Z>rn*$w))YSOYMFIw-3H0PSkhU713~k!< zm^yw~{&}=0#;W*rPJrE@z@*cj1wGzYKzs!kqPOS&{ zu8kf@Z<18)6Mwu*g>{}3N?@>emX1&n2r*xUU*Q#{=Iy=pKMUAs@?SOzB}m8u_&k`? z`zMzGsY*zJ42%({BvC}++R955>7mR zs+H5|-U^bw=F&C@>iejRd@43W)8@DJvWg_gnkCS;%QdlZA~AoF++5iSIa{w(vQ;5Y z^$D7p6rZ=1d&|=f>o@jo24GjQ#8_c34zxJQ@!=}W*vX>Puj-_6IhYz} zdkpOrTT(jKswJrI1+k%;azuQ6^oLiSM8?ctO3;w1L>ON^9A*_NV`q1|{}^C4J!X6k zT9ScdS-=q`T)BK*Iq~e)S_oTy_|07^2w0Y1)nkYRE0y!#30+hWtlQQ~WZBMU|gkr30s6+`e}QuL$7@XhXc_ zqC;5h5q=8EuU=^bD%gmxi|=vVIU|O|>!Uad1e-q_aERAS$!#86KSWAfP7L~Gf8u}A z_9o3$2Rl1MmrNNwH~+XNDoIp|S1oHBeU@TH8Xg(Z$;Hf?kc<-S_4hZAjHw@FY@k72 zWKCUqStJKKhyityFf6sc^7Z>PmVlrQsRI6sK;nZ3t~Y~()0nA0j6GQs5wD<%E`Hse ze02JYQm-v@{2=sqrRY`v2XI8P3*Ut~FZGmWTvUw^_zE3&#^ua95`ab=RU&JJ*q{9; zauO`A@%bso`+v2v{KI5a0wW2d5_faCXs^n8ZQxq0A@f1ZsoGku72Vg?D?ik}PZe`C z2ZpKqN;awJ2iEnk0> zPisai%24wh-fnnoaOiwb;SPEq;itQvT2`UW;8@V~*K%f(Al;bHGY6PjNRT?Doyy(9 zYzoACcI{$pePYTSl+?9 z>;zmAK@&Tu9jP^=HUaVX^xx5l&YuH zDa}vVR>H-bCvP`CJE^j8v%r&23_;rJj)$b1`#60}9OwweJox8jkysBd`%b^!g+IOz zo7Z!S{4JI%H;h}C@)V8AiB)70AzA`3gpIK>s4K(JQ{s`@?`Eu_l54}JLcj0EYJ}m~ zb4-VO3WU)V4qR3w`my-u80S@0Hd#X?#^OzRy8W6nf?c+COl zZ_2h1C#FNE79)}12+&q2j~Fk^?n^lpouhlL>6Y4-Jm|QM6IN;lI)=#S_{q}!VuSJg zFUo)W;iT9%{9uS?OD?}BPTMJeT_$k&w#3s;H8K*^M`l4OLaTETh`A>nC!7nX>jvvG z&t0(H1;&2gCCLZAKK4jA8QGsNpMi8uHUlEf5`Y{A1_oiXEvU9~gY1Z-83J^A!n^gv z^z?74z}{ge?xrB{ZuFXbHa^|BXMJJOhVDe;+z6|o@^qHoeL42DB`PxKk~7Nw(zs@| zS=0UpFc8HwF`ORJ(ru4jX&Zwxadiq>JJ;-gXbp8k8EdRoV6~Ds#&}C}#760som_O( z!tfbUcX# zz;j^%j|X6u2d+68gq`0))lxVt(2md8;6${#HyL;c1Ju^P6I7qS9DP2h3h(A5-4PqE zff%JCVF z!4@QI$Q$pVkNQgfN7hJ6KE3&W*XNt?M|tqVeEk^)%LDGYu|%NJr4@QHY&hW~xhV_h0wq zo6jS`fgrI{PGQH)k5$3SJ|LI3pX&=Hkbn5p??$lF>MUzc2;b_W`kZ$G{*K=u+yF9^{Vao5Ko$z(fMq*Nfwn3rc)WdzWuC6L5w;h zcfH>xs*c*4DJ!UoRY+StfiYt9r3uZ7$Gg-z1z8_$m8!+pEXQ+)`C#G9kt00>V)CxJ2h-k@tXYu zYNoZ;m6{mN$!(VXxsQrhN~F+cZlrvjF&~cD9bv0c7Z$FD!fiEC`s&NFCsF5zTh0>>cIe+2HQi|M- zy8caO8;?{NhNHCTq`jJJq-AvNEH0{6ysA|DbV&hZ1?D{hw=4C4$cnV1O=-+AOh;9;tsD<$iRA7@hdr=n0*U9OKOr*DS36p3SU`>jE z9%Eyji(2sf+)|WfAXoDi2YfqN63@h1UMHtf28;v%YJk$>T@3Pyl+>e{bUaX9fX<+meD?48DK3fTv|+b^bh0Wec4y00H28 z_~+^S)59*WXD5*4sA+}v`ig6;k-EHl7d&j`<9LGgMnj<7swMwub2+%1f znw3`O%O!`?Z-;_pwlP8~H7qCc6birG+B}mMLF>o#V!{d4<)mkweV%yU#}G!8225(H ztc;5Cyp_8=1tnS>Iz-%I(~sUmLN;_q4(qfOA5#wvTInWn+0E@x^5TGhO2X@6nQaI7 zeUftJ7#))4G1!)5MfooHX;K{Aj1pQb;>1V7@>F2Lxj0}5#I2Q;qRc4H@K7$>302iB z8svC|6yMJ{(h-w8tTnOOcph_suXRB8UYf41o5mz$caPkgiR`F*TI)IISI^q8fk#%) zaeHtH@X~~j)v-5CcWQ+R+V+#PBM*QDEF4w45TReaWlyh4xVPBJx}1?i z-VA(+>SnpB^~g8m-Tz?C55x{?x^Vm?B5!$>I3nBllouD2QT;3DlszrOpUyvTLvQfb6dz z9oyHpgYyYV`@MU|U5Z&7@-jT0krtBLbStxt^sSLUiDKF)jkR1t;CIyKtx04ZNY->o zgJexAR&l!8$maRjQ~U0#_e{#@M_-t9zyS z1=1L6C@G>|w2`F`=QkxDBli^M4{g0ctjj%+tfBv3St9}VkMS%0QJt&pNKQe>e!^yQ zkUR_qBx`c~kW0NpCP1==?~CjJsQ>icl35jfEMW`1y;ZG7gRV(mS;{eBkui3>1~p$O z>CgmM!JZpFZ9<_1H-eRSG_Nsgc*<*0K48|9ETn*z4U?*(52Itam&Dg%`mra?luPV= zF20xOAPJvij!N+YrhYgg`X|Y}4y)_-Z=t8T!i#wk)jn*UR{YAV25nK?a1uX1^&bWC zuHTdYHu55Ib)EIj4&w0v4p2RU`TmtHsB@Ihm-YcvH4wRV?dwoA|3F<9K) zL=(Q%bt#x6IHN!pr`jhmwEtv{Dq8t}vgW6m1ZvunW9CsM=bZupgHb5P-Tsf5IJEMY z`-ViXxS)(NoY)E2Ku^E#?KWus%wIuoP_YIFWrY=_&7zM2*zS8Xr z{C6CwRSH&HxElPRdl^}DN-ZAfP-J^f)arlj%~;Azq@&Xbl8Mm2DUy=gLs5aoO(|%5 zVv6+mA}_jnOU}-+VST^QxNN>zT&} zQ{x+olfov&Dig%At4)cv#h6Q%vt%Q@P)Vx!9YX?|FJ08>dr&*KYAdopplTt#qG}T6 zy?Pm46mu-4M2<$bJMutVoyqKjDLftZIuo-@1AO}>&a6S5>MYnR83f*o@V;h>MB31! zIVG07-`{`uG%~m}+3vWc@mY8hTR9tV#z&F(>qtUDx%h0exOtavAqiFLhvM45if1Au zoV|@;jFT|b8R4TBS`>QL)aHMf%!J~qk+AHe#!4w-)mnw<*L4Y5(6RDjzNOSRH6UH- zOmo5gY?NZY3ktJ*%#uYz`f^bWZTWV0`1^E+f%cSpWp8r;dIEJZ-MlDE zoL{0S`t0`$B`(#l-tYuAoDGtUt|q+)m1o^q&NzF!(9}kdfWeK-VRXk*mf{u-DAyR!*i(p!)mbt;GQ1)ml4-RPu$&H^1;o>(UnF_&qcw-#`q7y5&3xI*x-%@509oG(k$?(ww&S2K_PsGH z0{1@n1oCxDf*AUWbBdtyrr1_v))&3K#w?*^2p!pq$S02bfB`vWGPRqHk_t3h0)?2L zOp1)LXS50()3pSf#aThnXm6AIs8YLO_0h%+hG*5~Z*irZbsFr}Jj(4H~V( z{nJ8D1bd(~^r2KwfnkaA9LU(lSFgK(Wgx7iR+oofvN4DQe5Xz?nv4~e5HaeJO+cVW zl2WwJ65oR!VQBt^w2A1XGQH^Yy~HR|HBZ#&r*f@W((saIh@Pq#Pw+7NdlQn~2}stw z_Ki5d#yd8nZAJYjYhv$W{*yKJ*&Uf6S(99p5nh&bRPLwGTn>^oVNmifS~KHypj)&y zALw`J@dS%k8n7fK1;yP#J>k8eq*DbJ63`Oxb)WUB0Oz(Iv2Ry8>vEGzt%=^GrX3u~ zE_TCMt2Oqs0A^-r0d-LN=`4-G z8UlYSM;`7mxKv7=@Ee$dg?R#O_k(j`mc)9{Ed*Z47!q(p1dM}i>+m0wC~4n^YQ!pz zcJ`}jxwttf0Bv|3qhyJ8%FqkogVEC$3@hm1eO$F?x*Zs$!m~Hr0t{ks2aY~iMLg9Q zwD4(WOau_7!v_>u^mD6#c-Nd5*Cx@Qj>NR-fkC|mA*GI5%ATPlGeV?$qBO$D&;|{F zY4Npnj9U}`W2dPHN(l%lb+P7Ks6?*W>vt}ae&G{v0vww}2TrKfn|4b#&{2IGEwX?7 zy(8?t>OwL&2?E&192DZpyL{nr5(PRd1u2GMj+V1@?2CQKVR(F-c~H!{?$Jj9N1 zAO&Qf;4kc9AdZGBuo(oNjzkqF5O3Fv$_&CH+S9jZ1R(D; zfI7j1a!d@JZd0V+zh+CX>SV&p&Cq`R{?6aG%32s>_|#pV;WGlj>MgGhNFZ|o5XBNH zuu3dVx|P9N2nqOxbUgk+j+ml~`Ru*4olw@V7fZy`g4RNo>T3fU8+?#5sjbJs9%%S; z&L)5TK6kNc)iMoHqiIHiLL9??`AaW6dyuMvCqX3HLX4{MsS^t|N)cR>)iA@{>1$MC zC!zr5@-8{o$MLE~k%uXPR?twYv^mW}&3~o9(T>GN4xbXOyQo8e=v*`0KDdTs*pCnB zz~#iahR68Y)OAkgCt`qgmNK1{al=nC@{4lh++344cN8bJe(kLPCe+ z?WPxYG^S&Q&Vtb2E+q!-R-`V&wpvSMbuk?gl7D>`Y+46(e~&BTij4J(&Csxltb1)0 zyR{jW`HXH|rQL$hOy{e^2bBCjY$%4iwvHuCN3B*sHyp^~fm4A~Ej9Y+?n3N4zt zN`d!AM z0L(ErhKN!T$1rnH<=}9a!EP)2UvkkwR0|40CQ(GbFpQD?-Y(WhdW}A${iV2*{~RxN zP+so9jT2K%jR!sge|$s`Bkhbh`jiThJgjQK!2?&@Bnc8|lKgI%n8qrTf^N0&hb@+oaPNQgv$M=bPkoUQ>H=}(o4nQ02IuqTE)NO}yxrn0PFb^< z2d#{17q8YW4iG;SYD$ug_GF%sD5Fk{jc0-nRA#hYti%nO({U7OfjO`U-C^Djtc6T> zGN()#aHe+Q?pH%iV4g5f&p!m6@koizD&Cp>z(P4>K8<8tt3WHkDXTt`BXAC+MSN4X zfj0y^?cs6Q;0J7xBi)Q{&1~9stIr;EIrj zbW0gt(kFBqyU2-u%1pWXVr|Trb3{ZxfX%=ZpK=xSF)D4E$xx~85?J*rx+s|zarC5 z`_1KJef06%Je0x7J8LudO|LpZeZK<|6JD=A^j(8`cY^;f0T@7q2qwFWD6ekkQ#se2a1@%7Zg#fr6zL3J5#ci_W#!ejIj_qOC5Nkt7&`7%*H~h z(9TqnIz7&}*a2T7X^12&j;4^TZ)P$SI+1?sdmCN&=ilBF=4nPNdb}sP zN#|C}F;@T752E@mc3T!~4=!~Lnnz#MqXgDibFZl-td!6ASm>@yx^2{<*f`CWCY*tF zl{+4V3mM3A{6vw#-EeF#-a8|5>Ogmtuu^pMA{FKLNAQp$h!1%mUFGuPG&vjKFeC1= zT3x~-mmNCa+-bNb{zJje&wEv*xslc1wwb0Qsk=6%S(0kjYUF`e^b{HB%AS&^Qos^1 zN!UHuPO~loBZxO;NfGqt{-LF?h}OVSsh_5n8b+Rb20u@PwN?JfcFBbw1AjAH%1~nB zoZ)Z=vZ~UsvLX8$=cyaSEo9d)mY4N*s`qcLpTt*0-=;qw=^_e)SpS_TZ)#a(DC69^ zMKbsSF(s?Oj7&J@Wah z5KVVEx>3{#jAC=ENKfp2EEKl4__cDP6&{@J@kWEN^4nw0nbsQ~TNE_<)cH0calfAz zn)h$L9n*y?r)S+A$I-%0tfPq;Y`D8y0C3Z6_x+Mo;1~NNd9i9nChr{~8zCTfIj580 zC)MhwLW~?2$j4_S>>sXOYtnBTN6v(gz10cVTLm7Lclq=4v1YY(p7k-bf|5Bqv#jyk z%|MA6`z~E)hxzLB)xRm4R-F;}mq^>rd37C(Gdb2aauy*hbc^C?TTznFQL{G?MvTAl zoy;7UOQBN_8Bc}PT*sdrj1m6+-4)msd?Kz^2hRtpEH64lEygRFdN6_j3^oYBI9r_$ z%~Rd<84&iAzXSKA&ys;RU;;w%lbj9x9tYp+|3xi+ArLzR9E4;jg6p8_F znoUU)#!MOLCxguAt+TzSfUNm3U!te=GX?D=t=MVu=MV0^PqO_*!-lCQ^~FCF!o
  • Vj;&ySk_p9C-DS%liPh$IQwZ47&w757=0yx$l9&I9%jjXlE?P#0 zo+6*1YkFmsuudMQKeYMg)=oEF;77mj#aW&E*}4#~v35xF;@^YHGHV(2r(VybWB$=g zvZF5s5v&PO(nu%F8YN>;cN}DkG;@Ht(*#YJ+m&|vap`L`W9`R zM%-k8%P3ZG0Qe2j_-hsvpgG-q0>H-Z-gs?Kz#z43W4SP$>-r>gmBBp3jLP2lp@R$z;h+Jl|2`_K zvFDhrWqT0x!kiVW;2f*xWDuEw_by9#6Yey{=F6a!$vfS@L5fXP)7YIfgYo7+9|)f3 z(nTaA8uyuQiVCv@uH1j~{z>!}v9n$zUOfJJA(k!;Y;X6hN)};zmAV^yzL^6gqN%r2 z{Tmk6E_KE?&^`kCoclAJKx(G}%T{kz9{YknzZR0eo>BA17Jxx~7P_tW=PPm&(=h$A zi=MF#+TFwP_h!Lfn4F%G+TCB-IEJuFr!juBO@+lH0qx|fQrnF-zh`|v4Lh6xktxzt zjmUL4ujT#1VftaoZ610TWXN=PbYbkAQU2F4p}mvzwuWWMP$#Z!SThW$gRjMt=t0$P+DA9E z3!(yw(TU<|TlEMILoopH)00UWqAfMm40mekdN^riBO-4HPi)gD%Y&uo5Zx<2Eyv}f zL~Gb|As8<{?M+NKO~jy-?9OZIzWMIG#Qq9q@%_w;@O@>qv{Q#RL8|i;F7BOYp?Uf$ ztXS8<9rk6zUnPuC&xRRU^PIj$nclhP|)!PJXx&uS09B%+umsMDnvnwZ}O!u=o|5aa^ zZcCf^QzUe$(A5s&PrbEPOXfu}@>h(~_e4P^?Mo5f2&jC7&CT2`YTd@rvD$;Jt}spy zXot+q(HvYkCgN3E{cWGa}4+_NPq8@60VXPR66jgIJ`N2^HS}3XRRh`LY<_4Qsv9 zOLS}MbVn~F>yH!s>Q<``tweeF0SN3j`7gpoGr8%$=^l@T?aV?U_p0q>*w)=cPD@V@ zv8^ZByYBvB7Zu`Feh&Nk4)}?wQMyPhd;L z?U7^(_9-c@p^%~+(aTIIHW9YUUv<^z`uz%Vnr!MV0ev=QfMgw ziEfC|21nIF72|>zUxkE$FE%VI-Dfgg^j}_P%85kt zDf0D5(|{rf+2DBtXGoi!hafL7Ms=Asa#dSFCae2R^i5-T zeZSVDqRjx>&F8`07X~TjhdYjwUpMCEOkA42EK8pZN>zj!zlVB5kv(6Xn%t3Yo06*z zj!jSLh;!E~iZ5w2pQ@2i6+~I&MP7Ymw?60m1;EDbOd;+^B!8;aG`wjnJ!qI%BEHOd zY;U5i8bd8{!__SRUs^_@FHKx%GeA-)2sEoP_TRO()X}K@cPpb$slQ6FOsF$uuBqW% z=ZU2OquU8Nb2VdBIpzec2|zvngL&>RrKYF3!twI>iu1BH1B~W#YB}32x$B!lK}Np# z+8x4y`T4aQ{@+tJJOA635wBAqA^&K*i};m^((tD66;JYE<23@*6s{xKjjkiU-w5!; zK&!)M|3C)UXH>M~(RQ40|H=!ZAt51P2v^85*r9S)hPr~=HuXY0mKcO@sW7B43S}JH zrm3zcLj^{Uk3J^=)oWqf<-T}%Dp!F%evcq9GJUFaB#8I58gaHsIJF~Gyn~)6uMPgh zJd38vJJwbgf#?-SZs}g?5@)O=j$$c&#b5(zHE+7T~Xib=dYryu-aF|;>6NfdSvU3hoR~M zJh2z|OJ51<6y3mWfb`f4)|!Is=(zevuAh!TUgFULdH5H97i6My0Yk~R=;)nG zG1H*+Or-A87wf{4Lb`T(h~gE<^`Q1xL4w56B8y6i^F1slJ>@Bl(a2LI3i}*=7-`!h z2(0$21UH`-XOYB<5s~xCi69Hq7GIf*xb;M@pFj8Dnks9ydB&<73fkqHt}`JD#tp?Z0> zpM>2rklNysL*4j8Q#Y8LfZlQ*=&l6tiJJ!zJg(>ID9so2dz z^nQfsT!`U>K9R{J^WFrILC9q~r)5y>vQFh70sUxi5YusNFP(x3cpL(`r;dpn{l9j3 z|8l$l7rv5#r@{gT!N9u1)m>JsdUd~RATINP`sden*ci#u++aizsY1lNpGfFYl}vy* zo*DBxW&7yHZLHDw-8zApxLS;QTDd?M7Y1k*W{u2%@dG&mT`ZS}dsU3fKp%+@P=gEj z6+*q_nPU#s1NxZE0;5l#KJt4&A^xRT|A;wkZiLAsY|V9K54>3T^d_#r%4#oEeJGPj zUrE(#GfmTYG|u1BA}_BJT?#pI&r-t41u=DWkkSOZFjhyli79an#n%eR(kYf94GX}o zLleDoKxiHzWs(Z*wB)Zb|6h0Kh12jCwuufB~Zw=lN2Z26*hDolR7D08={0$+jyqFiMw!V4oiekMfgNpGY-}iB4WxxNMyxHbIO_29YanZs#S=Wm>9# zrCdw-I&oX?H{}dAqn!s%R08;hB$OG3*-n(r2QkZUg(Koj7VUyi&}F%mCi%)yGwAFW z(<=dnBNYr0t2ssrddnL%~jeo$U!wspU82kl!~5Mj7a3)NpV?J&Qw(+uTy^aBhE6K!<$ z*+B-g|MD_Yam?XblLh#%A@ue`@l?IUJz&?=iC~B7>Hu_!yLZkxWd*-}eC!41%-<_s zC=KMuLP@Y24GB2r5n^e$={aXK!mNvB29uz?%qZz2E$}xa+>NK1uR1_v1(cV`QMi!% z%Z1)H`m0@&6z1H$IL1+$`!FIUvCNHz%TPsO7L75OtHL(+e&d2)BfDs8w$icN4?bs+ zz}SE)s^2Sbx(bPDk~B7m+}Kk-Cn7V@{7!x8zq}0Me|edS(GUHQB-7?2fXbGo8=OXU zFhtOO)s|c}C-OI3sI|5igGHVFtt2*Xunwv^I)rFk;+$k1=Qr+*6F922S;o7U@CcyK zN?*Z%qBNnXv+Ul6dW(L^w6`?b0GLc0J9(RkFSvAEyb)K&@qc-l6=TxLzqme=^yuGD zrGJ3&9EPF>6;3Or6n`lkp}+gx=CUk3YbWiaUI_}mxkt!aFY~1fRgm=7` zFQK!tCzAcusHn$s7m&x zV$iRc5_>l1O4Q^51_hG!@ZL*HCC7bB=$L{6E3tt?hHgDlz4-BZs%C50Nd`vg1k-a| zOrG|2q56%(MRlo;09-avVsZSCI4Ok#Y>|)X8D)t84B-k(*nJl_`Kbv;ysv-ad(~h$ zUgiv8DN(WD#b&ir;ljwPBAkmn<}9UVcAVG0 ztdZ-*_Of?e@}osuDXIr93-D!7rGm1P83xoF0LuFDmk!w`sVJ@kk0JJN;A|G8(u9Nd zMbhtSh<(5OUhIAmk>ijXL`Ox|jB_=HqPsUHc9N-Plw_ugZ<`qOMx46d;`dj(#ib9V ze;4RNHtWzJK`-arrHN2<@+kLOP!wgnjo=$~v#DZpX}!nNSDYoCpoJUtwMi6nY;!Gj zRH%azuL?5kk;BEi)kPlZ0rh1R*Lhwbl_M*Kl%$o%b#hB-2icQwo%HQCxFL6Y54NuD|%&)Q22ky`Co9IK~9P73^q^H8D)X$8#|T7bkV&y%r+@NNtT zjm;Sf=Fa$&XhQZ(Vscee(O4}Vk;`RD(=-(<+1LZ)08Pd8^zGzPI)qc{OgqV(5l^f+3 z4~&r4Ibp0n;KT9Zc|LUoIj5?@4lo3uO$Y{AvDVm6xUo!y;RnfvDX1Q3G&+>k9JEeG z62*kr5B@e`TV`DvJ1ypc_ZMi1Rkt3YbFKZdNF{c%x>+IUMR}6$d0+PxHKNNaF%l; zYGThLaLO879@&<$g!<3wI0Q7Cbf{{NO?s(Bfh`w{0H`m+URcC+%-Usdn`{k_FLuTd zE=u}~fLyj1y&0+HlQ?4UY7-YsaNU@5H>yFq0V1xtPl1+87qOMHhLmCRoAz2t4K%YO z!;>0OGWc6ouH3DeGn~`AGY}F1%#T_#r%@=<2>e})wOLs3qOoIrpV%vMRwX7RS*fvs zb#od0^`S?r=tW9V(RbQWHjG5zPr_<7fw^`f;%%S@fqQu_%iA}^q=Om@E3s(EYWC9o zGeER;n9nd2Z0-^%(&%{W3>}Xhfjym0sqJ~wITh3@&6)+b81!^RRW6Tf05V*_DxayQ^<9Nmwq}6mFg%UG$yBree1z+*Bu!{)O)F`#F(9w-;v1} zu^j4vhy3@gAq2^lKnCqWD*NRitm=mm@@qIkL$}o*>BRhVq*m15iC$rig*yj}(RF{# z4sgOmTO_eX;h-5=Dr;4wq2Kq~3Y`z-(jmEfkDsfM$-@UPRHIMp~q^`TITcbs%A115q;vOeJ^Gec8YQ{E>3Nxv{*58<;Q+&E@;zK2k>M zXOplHh%`~=1~5hUxH_-)y}vIU%kf!SU91e|0IP$4DR$PtmiN)}3tFBmD z9JbSnH};L(i9V*xfK`zePBoBA{P6=ySJq`|+vw!?hPPw1@~g@VI;DFzWi+ zLR9~;)3c_!)#u6<5VFr!$jXeTe!a?qvZn2vI(eQT9JBbZ0!C?J1>PgcJEbnD6ubBg_F$x$Ltzp{johIj8 zTc{7bex#!cq`nhmhUQeZX61_1l7h@dBfFJCl$1HL2wl+PgsD@V@M37m1%7SsiT@=>}`)LGHLZ>CEOz=^*PgC8obx6YO&XXwwe?T z+V}1KS{v(TI($kAB_0LNmGO$IK2d*Wds(K=$y$;XN7WII6f%XiBpZU*HiCEQ$Wj8U zjXWnUEm@`!I!I<;k#v~=JH4eMQoZh|k`5my+8;AH#RjSTlfq1e?$v%Z*oP@86N zbu<)aP)L2+jX)XvU-CE7W>H-;7Q@Vxg!{szOg5Fr$4+$sYcveZxRTVdTBzNg1J?Me zz6^&c&sCdW*KZJ&kSHI&M*KK)Dzblp^7=EGGRvnf<5km+6zk1!tJ25G{8H=QLcyDS zR0B<%s>=1way*W<4no!jPg22Z8RJGEp}+t$p@Z;*P7GJR@Kx0P5raEDU-w-&8|5D* z2~1UjuSXZ%x;*ruQHd7bai1kh^}$MN+bl@CGSQLKc{o-z*OGh<0h%Bdcn7$Mo0c5; zqU}G7{9y@klv5HYNDtg4xO}Z{Z6R*J7*@vvR$pK~n|b{zeH&&M?^t9zLmO;b1R|Dl zy+%M;)fW696WEG@4$28h4W|haVmQ39{J8*Hq{JF{?yk zg(?NPp8j;QQ3D-)ZYD{Y2i2ma>-9=;$AF_Nq7M~eqzY4Z$E?GB4fw~A^uTX=cj_V{ zLlp>N|{&MEDp$TOi{i&Upb8&rR``U6Zw0UTv3IHRiHU)-RA59y^6P7 zIueid*oyX{*ySdNh6&6iH;hXI6xESJ&s`9dDqcjsY9|P21Se7s*yiHG*mUMBX<#}K zATUWb7~`z{?QTN$I}r9lwT9P?bhlgPTXFB7$Q*{mWx6v`;S(72GY!y~x5lm_wU+fy z=!IJfr4am#;Wa^Q2k_|nuF5w|h|amIK&o5O({Vy-rimCcd?25(x#Zi35>l>WV+qI@ z{rt6ERW+(;iSdyMGJ*Qb0<)GUb?lVAM>>2ksV~4#iee@(I+Tvvw}CQJ-@6&5oGFMDg@wU35-9B;(|?5$YO&;gEzERPq4zh`$OljFnuCDKyB~g; zf5=K9?k2s_)vr+jSN*%b{D3o|-W9J~fWVNs^ijKGzFe5Xw{_YR&B!0BXb74S!F^i zJq=z6!+YN0cyQ+G@Q|N$U5UT#=u1})`L~t!&IArs8&yKU>v}D$8sBr|nk-~b9$&io z^!n{Q{M9;yC^AY19JeLg@e9n5(}VR|HP0~%eoLi0ud7S&Sy6o@c9w**N*^cArEdN8 zTOdSnKxj!1r%TDi>Q1s{UkYItH}}c>!ksk%lF4_^S>BQv5_4A;C2bL0Cyo)66C)>n z8Dw~UDgf7=Bf+||^dNvVf8;yPr|cX&E`zlyG5k__2JWDRZBabWfq4a%d1?=ON|d_@ z3PkAdgelFjG{zFIrOe*JvTf>LQ6O_eU9h6_3A@W2)<4xjJs736%5}c6LEeRsBS@$% zPN7K5Gam$t(cf#Y~*T?ifZ$7p-0S;zxKZ5U0agY24yX0YO;T8$4_NwF}Rx; zSmZWzDAkAkTWIMp0RG{f{x7jDAL=Ep{{F#QbOF>#vd4LeuDp6U9YU$S*Jy{AM%OKZ?=J8e6BM?*&A6t&c~&`s0mWlG5TIR#I-fDs}Z=WE#gToX9_cN4mz7-+4@ z*_FuHLhpVtki51`JhnqNYj@c3RgR@b+1Upf3mbcq0BrKo+GtL+s*1q!5j)V;>=lGQ zv~yJTgYsYMU)?2-Cbw@)PYb0)k+2i1rSf*>zYV2y`&P(nof@=J=t!G1*hh{$C24?GCIG)qx+$CV}oYURUDNG;(V4?iYKLOHIl|#NJM7-6WPad zV=brNK%*h#O;V0p0*vxfr3r&}au8(<5!ri#Nt%XAy`KRsv|lWWh!eMcx^tmfuV)gY z2rA`F)6rz(bxPUq$=g?r%i4C+OV$KNZvDin!g43I8cMtK!fCu)sIf%9QQ7GCKN71d zPM1lKo}(5v=Zb3P9F&n_2~15VT7pelo?fKj=3uL#lzzo59F^GCY$(+^;+5VA`npEH zH*dhASDhA&_ouv~bT74(pm4$_?+d_+nR+T{QMwvQB&`eJ>v&&y35?>LUI@Mb?HyqE zWj1JV-jtu9*0od@y*oFM8glpAr((mVIh9;7kV8~VXEK)9qC(j4c*581cM}wX!hbx$ zbk|cJM6rRHC7jZCmn5()LR1C>j7~mla$A1XgPXi4o76^-m%5nYq!vN7LH_5b1o&Um zVlfmJF0xV59R3FDNE@u;wEX=K0arfDb)L(n0YovW8t#_{3;`*nnlwc^Ko($f<=+Hf zWlen>(q($;osGf4*cj~ z&A3CJAG^Y9_k&}nucb+G)IzjwueP#tRB&^;J*s=D%>FUIh0F<$?^W9u%XL*oH*0Wi5?yXGFESS^t-tZkJA~9*G+f-;Mtp)N&HpSf?T~H|< z*6r^nd}pqxe!hkn#%zw3JR?TF23fJ?6Wf;%8SCj@|Fm4TS4D5~JWZ&>19+*l!Anc~ z170c8B7$-``Ngg79z>@{Y>p8nxN| zgOjljxo`#(IHoHm83p1fRuw7=?D6v0#4 zQ--&+`40Gtoxr&Ci#DEI1ezXNxjaQIZQ`s(%}4p6K}Jgi*Me(_6MMw73ZyNjxv9?% zu~S2^rm2hkp`Nx+O)sjcPb)%G4_${f+LO4+Y>>Qf9bEb|LMfD`Y2MQ>98T?%oqH6a+a~ZBV>BEV7~;~_ z2YK$h!F_`?etzXgdAT+Hvt9*1f}{5lDB)9JEO+`Lrrg-d6#inyR>O^4&4CnIf0MpR zn#=jhE(N3|P}t9SQnrZ{hO^Mef$|XZV?n^^w!nweXNxlSw(v#s9@Npjx57;$rl?~7 z=*P>uk2%U%7m`FVK`*KvyO=z8Z$@yjQm%&JU3alkTQlEl2yCg?zjeqe!=(l*)+I^& zckCiP5S{ zM~XigsAk&x1$WSgk;FLIlXQ@$7|CU-W3HH`VQ7P#M!ea~|3=Xg*<_D)0$w`=LQM=!-0OcH{Amz*8r+D>!!UVOqYDMzV&Yz+WQ6eP6h z?KnoF>4(mJ#2aPcin3pwrLwQil6$bM4CHqlzrQ%iKL#bm|H9#X-zAe96>q_8Q@m@h z;)59feXYoun6rTuLTdRV_rul6^Ye27$$l%>*VFBN;a{4Wi2ME1d8?n`$Hsu!7^C6P z{z$sG^~c8WY;Sg6c8_NZYvh7NJV)POastu0XlSxN3Oh6@Cuyu>76Nr#V0=3~Ezx&0 z{bzsY3|q4pL*B5?nbw)*PMIx(aYJH5qC24MtF#pI{!iYB&==b123T5B;NZv1^p2ID zZ$;D`RDTO^pxbBw$tu(QW@-@9C}cvZiI_p)GR{5*aaSdPM!5QX`*HJJ=hI^DM z7ejFH*wQ}Wz%otZ2h0APVej3Y zdN~W9zB|>(H-$hArPA}V%IN2&yIy7bSi4=(Neja=(8ze`dPn&D1H+?iz+VkY4sD6w z5Y408CVxqD`Sec?o;|7;%|bf*7>+7ifR1!jneD{pNK+Y>Z1k3+Mb>A1168yyeUyI- zNL(Sfsj=_yXV21~mKvRl-IOkgt{ToV?v~W)x|##Mo3G503ryxl)}Tt<{--tw@q9J) zJRrWJqprgG`rjSnb$cEF(_??{e^Hp5UZ{`KS9zq{_AJ0k*B}!dw#8kPUXFn1v;98G z8w;V)L(V&);Pc9DB%nS}kFy6;pZ}m4W>mVkX3J5&`I||CfZh*^~VAN}nwn z^J96hMVV{mPVEkm=qBgOY63qs%%mKiUy04M&T|D5a=r*sH$Dn)92#owTd{t+x8nk@ z(G=byP%^W8`m<{vvwj~%q)U2l8#BgU`Tg}Lf=;F9{?-03@g}+mEmquW!p77}8U3dy zjmG)K*BgijUs_R2I%R=R1sQFabxq<29wFU!A@%f*ZvLT}v1ZMPIyCH0y6FQSMH5pD zvvL0}!G49W#u5dk&Dlg%w)3HBYnJyFB>)D4jHmT~Ntn4iU}fnY$UDAu1IkJYtOdX< zi_04$gEAlAnP@E_QF=Cc@)CfJC4;ZVk}pKdLLzfR3gBct=b&{P*jOU-Xc{ZJupt_t%%h$wnrf_`IjZeJz42I-CEa}tB zgLu&haV*@AUS5Ialo_RyaUl}!kOA*SxAzsyJcfy?{22`HfClOVMoLVwNVuct{kHP~ zKNRoL-uIt#A%F=+ihTw^+{~x`F+NvU zz``1dA50`y7W3`V1GV#e*Vw1r;QY;gIHazPo{68^yETRx%7wt|uHJ6r$cvyLzwgIF zxnRt4j`7XQw&1zoc5NPD>#p<-_-9HxW^XIjna(B z#X#98t*~u^V7OKm)Le^AkDYP2)Ul`#rChu_FD;@w?=;^RBp*8t6{Tga3u!gGsl)zcg7Qah+D}++Msi9bzD{PH|CghaX#g8= z7?4;Jhj3Sx1YrPX8(r5vyWD840<`6dtJgs`1)Q2D&r+g3DKTq@CSvjj2I#TRPt8OV zP7RWrlI+cffc=DCQ`ST!>mJl^)bZq<+y%1vG_;tv4CSlfR2;d}QW!r~0uhm%UCRU? zA9&k|=?5*!Mq;@$WCKYGsbk0zeqjuP6~h>VvFTVI=l_}YfE<+oo7)z=_41v);>>T$ z)K$9wNvn}MMROB3EN3Poux0uUHs)DXauAh4gC*HMGn6)eSudP$nF1cYxR4_LjEg8B zT8@~+G(y8j4zA81T&iPX2bNY#33Bgs(U3o|vJ`X0H5v`B6O&1PNU*kvE;Vm6H}C3J z08O3mqoTl}cvRgsa)p9+DteiKT?C;uc5<8zyg-wci&9jRNr3WDP?r3ma4gmr7*<_z ziaZsR{Z&~yps}7xkqA8*gY%pwA2g}KNga+x^f~tXJnID-U@k8^tBQj*b&tJnOA7fS z*Bf;Nax1?gO9TyB{t5o|bkPAUH#HK+Iw(u^0;xF>$}F%o+&nM%li0$-Fbv3HgmhZ9 zSBnJ%2_78Et6Aoa!qiI~G$~rl{)11r-^~vYL~0}_3PMf7^cIqhljWHZ<~8HVoKY#* zB-;D(G1hinzNY~zO9e^+fCMCpKjFbnnkMROAwztNQ;WpAri~7PWBAjsWrc9YJ1H<- zFeN)SxD03bAPzKRj&17N0|W)8CvM)c)FHIQ0?aHWo8s``O+^hZphXbE7#7x!-USq3z zraDoT_8k>3-$k0#iFSxG{@@uaG)X%)(6m9B920$xn= z^h%VW{t9Xvo7Jb592;@&^RCD~tgTpl+8HMK6@-p?Z*KwWJ{4ZqjY{F8v5cn)d8u|d zwK~+wt7;i}WH&4?ZwcXqbaRrd^IQ&WhM}Xj(E$=Eifd?-DDK0uNfm*n=^rE961oUt z+6L{AsEsRKM?239nBNQ2f)AV>(F=&fnj4u=vN*^3Kk07wRqKX1?NIZ242Y?=SIWGsYYlES?bW*L&wWlvka$1RxfMT$t0 z3t(0|Xc@dV7DvCB7ETL;Mn&g!vPBh#llpSXfDth{LnaOppL1k3*^ggu9EWW$uMj&C z-!_+}uSQ0bYEv0f;+EqIhiGPMYA*6W9!9x~L)C&zX&t*`d%x^8o++eyJrD<2-T4GU zqq~J=Vj{3Fg2s&*>$JA;u$t5A??N2HCq^e4OtoT2g1bqZ!yeyjDKybQFHZ(1fXp5$ z{EZ^ALrkd-k|#H{8{?%C$B}G*UvBoJ_ zIb-;v899X*B0zp_Vy+f@Pr91%#q7WUYXR2|5z)kLF(vI_$IE#i3k8Q%j#h##j=Isi|_kp9QJ`$_u~ z?1YCL`?KqRJ4l>70Ky zx&(HX;=Vdds=}=FrjW2i161Ps<41M*14#;c8WL_8+as`)q9e?@I$f%@oswJyyGy7n zn3g9BABzNJ#BJL2KrR2mWc)95;!qS|T-5*i{V6iZxVW-=2S4cu&>1Qd})t#ehCQ*Z76J91we20#GqU! z(VN{>V^yN_?u@;j1qTie8%WQ?HZ&mQhsdr0GfS;X`b*5z`L%1~;QAD4$gPY|@^#2N z=7Gc1aLHH{b@F5)xc5a2Sy}j(327&M!xCKB(qDnOhJifQ!uP6|?|)lJts_)M~8ws_T~K|CUn_OropKbvChP=>tk7%UlNLEI|Q zMkQo78O+3|1Fa%WyBilRteo65^rWqLjiZD~RW&E92 z>@2KeAlF~L;WETC@W?>ZjLDI2*!9ohB@=h-lce~Rq?^FVQa03tX?{KUX`} zpRCBx&wnK2gZkC!;G9uhAHm;O^*#RH@?gk zf8b@%+SHqSp^qHgF5<%zenu7O?}Y)^EginQEc z{R2us6!8aia0#Ud!z}>1|4Ke@_y(Xu%-BFyFf31$iN1*z{#P^gy8cn~@$>lPqz(hn z>sJT3|2+PGE`)jad;E0mwl$VS9u;J)Da3RcN&E>eHlAqwxqp7TIuYS&^V4sfJA+Ta zmkik=GW^Fkp`b($x>?rkC)mfwBPm~b^#|HNJ5i#O<9IhBMj$b#r>B>vr=HF5?{G!< zToZfV*U^o55_Ht>t7v7su9fQ3usnT9Fsk{vnZGX#C{Mg-ul)9*^?YNdw*epBp9`L! zD>)zCVIO{sy|K5bgXJNlJv~S&2ZS0szWnZY9lR5hk{XWxwan1s{03TPU?;&5o@8aQ z+$AcwZu{-w?h<(yS66};SAu?9+a90Z=Dd;OhFGDU8@sJubDxbUA1&JpKEL#yI&0s&a#yZr zMBQ}yjVo15?CCX#8-GSx7t5*3LOU@C6`&;yJN@wP&28>{tX^pcY=o`c@aX@8-1~Lu z00q)@^+W)wyLF}u?&+PSo{Z3i`0w^6b%AaWB#5%}Z?g7@PECp@vGGDgTO{rRFej!e zxVnbeqD>@)v~0)8V7}>;38Sr|7dROYBkg7Ub3c1-isy210O$fH%xC0p2HSInF)qFG z!q=RD3#DQqm8P5o`DcH1ei4qjrvRjDG~Q2VM||fQGdX~Cjfa#*vk0zUUtM*;EW`c* zU14?w)z&?buDMz2MW<6Bz_wYsw}m`m8IU>)=8JOTopsI2dav4*hgAT#sw!XPaE_Lg z?lk9ZH4+yUa!QcyXcVVZpBzOu$O{CuuG=xti0hd)rV9BoAi4N$@ zz{%=YTvTwX%ibYrb&@b*`MHp@%xx&q9mRsjg1!N>#+;rV2eQ+F;ie5?shYOY;u3L( zXIIe3c;#YEIret$5t-DsT)TTm!+yiQk?P<(t@(S~g1(v(SZ&(g^g~3B6F13(35yo3 z^YRdnNOs4>Uhlg)s=^t$f1>uY$5G`z z19$yqH&K|!SBb5ab}xxP^IV~Io^eX=)kVq^`>z@AKVVI!<59G65ONw+QWulU7z~0j z1ti#{2JHO~S%S&>d!ayp^fou;htns3^Dr9R)x{u>Y+!>=`%^d zrIOj7Th4OJyI$LYF`Rd7G6M+KghjYwDfeg;0KpoUK_>i_Z#51sD5G#!VT`gwEaztT zy$L-MlB`ILy1lAgpULE=VzIg8c(f=?11f2C`X+e;zob8^KZ@a-KEVkja&?CESCL>I zC?2fF^7!R9p$=a>+~mpRB43k%U`_ddz#7*c@jS9wAXu}?Ea~zf)}3^{+SuK;b9A-) zTCan2-jK;P&BBF|(>+&86l~$U!7duT{^$P|%v=e&i?;8NG5jCF4EL8{CRlY>Y%nx` z`%5s>t^c+PB`oT}yzlLgKYitqc#Vr0(`m^Swu8ig_H5e@dj z+*_jF^!F81&)&T>#7hrLKSXl~n?~GK83ikhTV7{I4uP?o%oyE#JswU}ktx2l1lge6 z-@Hq60r>!<+?{j8j619hr46H=#Lpksj$vDl7#@#MlRG-E5wRxh7wHeS76EAS#eU-(MD4o`s#Aq z&ad+VrQ!P>>NMUtTzBmAIPbIz3uaevCdLRk_bTjHVaW+tSXv*htHYTOgZddScxx~A zRyzaY!%<=>B#ogmOrm@Z`gXPOkpo;Cx-ULIgFc=FeDO1mM}m0p_dx&X3C${kPsp(y zG-+1MnnYyTYs~hdf6EBldT(7J?j_UA)@OB!1d=rdp7MMzDwbbl&AtY$Y(K#S60yS< zSu?pW8_G6aJ8ZlVpi+Dz>FkReSA3czW{e}RY#BR-%}(qo?};ay)CY9TbOOm5#+8yR z_}rsD)|5L+^Q~t&tlA#Keoi_A^|_s_ zRX2>1hRA5Ub{-?f5}acpUclYQ(5YWy!R~3f|H^2XOfyPU&)eY z^b(z@!GVW8X_elip7e1I8R(8$-+mhsV0XPZY^}n@7>50|w~nI)PfRo&1i3!F*)1*8 zQikY7xY#h+&mdAIZo8$uP@z_{;N#5|<;)Q>#83wpMY;|3qE+GHS`b_i6$%F%yOo$K zO>CYFsTu~wvY502d^$H0T=t-{Nk5fCF?5uV94lQrJA_S4v3xcJtFpq_gG)?H@gs|> zBfLB9+R^v^)YcR5v))kqoTJSV9* zjc0(T0u#P%ag>BG+B$Q2Ix4lY1oA5V>dtE5`*EF|zgS?#_ZJz0n41^$N_u6=P6s-a zT>II)LAsu;ZeR9ndC$ZHI>4qc&>pNYKd>RjHHF26Av3IfZOlG5r|0fkmkkR+CDvSW ztmY=8^YiiLJ==YG{nhPsY&W}Ud2ZEm2O|bb`wd9e}^zo@81p7#%7RU-0w^hbc=q@wn>ozC6A(}HH?z#xzy|0V zJrMu!V)y@TXDWqkWMW^IRAgA^D4UmAjKkxWnXj7Aah7~5)NB)W$e|A>_N*&J7U%#e zzdDkgV#)Dy@bVuu)G4L- ze!!)dt;;Z_lHAYL<$d1H#rs}Lkg9YI!>3234PD`lOaNr}RK9pgXz!eGDXP;0$zcu}HI* zIr5AcIu4l8HH!!*p;)Tf{k|pSvVAJ@B^oXpvI~UHA2NBZH=1T6P7IK4d=3Qs7-#iidYWxfM8Gvl`__SKN3 zJCJ#xOO$A%M;~9Pn7|@0>Gi3w{H0kT9HNA?1}1!cdhm7t+UzRP&h6eSyW72q*lO8U>#9%j zH!uAHW*kSXyzlFi{;aQnN4I6aTh*6irqK&grRy@(QOBuGFsExHJB$SYNjA?Qh^ogX zCcg#*Yldj#s|LNpRA5_U^S>HPYCUiUF%_|jS>tR_HJ%>PEv64?R48Wdb)%rCI!xDf#a=$N56@F3_vAp?@D9B{sz z#e}p;4EZ%4S?2Fr_V^>G{|q%8?tBM1VXbGn4m#fl?Xr_0BY};j zlC8%A;_q&#*!F| zSXy2ExIf8$9Ii3>AB$`>Kys)ln_L4N&@q!;HZ1cW#|+kD%;b#yk6lrm_@8-t1i4+j zZ@VWCyK5UkoMCxc2*f8>bEST~@4xJTikWXW`5+4x)!R9r_jluygTRL=E{PHK8-eRd z0<-JCikaRB@^<2A=M08kj%Z7)Bs=lfEO9~DZPku)1SdzvBMo&N`5%Btw!teh`G;3wH(Mqs1gXlw&yE>%@)okQ%Os5+N#diB&lAYm=t2Jlbj&c< zCg3Bh&;E5^5d=DBg1;Oyo?!@)oMxDozIE}Z(@6OG^u_Ni`dngkUfbNmu^qQvYRn1F zo8oh}x>XO6sZkF{f(}0rJ;scFOp=yjF5x=@!5W=1AXsy|+Rdjy`Jht5OqtlMIw$J% zTU*pt$qs74ao9rXJ~dzWDekwJVUsmCcoVzEcXAr*M%qWW?8p|?!&3%=@w@*-mJF>G zkt(GwQ&k;3hJH)_$=(9(c;z|jC5I-N{)#Ns*TPL^LK6A{zMPOG(bC-C1pDYv|@j}_eokEK9Y%vG_|I+|P3ut2i4Bkf{E zU~?OKi+^X7_7_=0nGmx=2ABqtH4>j*k5zgAd%(Y-cQ?V_S`vWdQ*Y?c?N^OQ9KHvP zYn%DKvM|fs@6!5f)-~N5>|E2$V68q!;jKT0*+-YpTJCGsAil_&3#B6qx9!U>vgVFS zxBtNwVwS!1`>XjcoyO`Lw|X8@O$>LSV@B;slUAjqw$r1Edu8$T6XM}2^_oG$<>gtK z*_Pbm_;=Sz#F8r3*;i$$^Q*E%kHnj>J3A+ni)pHNsF;g5)|qzlRauHP4YSQ-p*+^{ zT)m*#?ws@sM1c9mAmotnzT&4A1z6qO+Vp)S0&I?6+1(Jj;&hl{Ns!9sDB~?!Glt(L z9_{RIRNi4{C^*5mHWK0QV;b<%W0PvX#MNv{3ixQj&q~~$#EbY%h7ln!+wIJJF7?LXex|dj@!tWyLk+*u8c;4eS8j^3QWRi{c z&JS$K0{&Wqt`$f;>}dccGucexS_0(1Mr_8qN*RfRHJ6s{F7}5x75rWz9chrj9l>2u@(hmfd`ytD5RgrhFaBu!E}9{(RlU2mA;vEkyack&UkUs>|@aHfWFlBcI(Fh zd|Enzs-b^+zW9z3x+*55zp>XO?%tiWko|H!{!sQ9s^7ib?7n4Xnc$iXA^s&G6vCIS z?1Ak0$}G*>C%t>NOdS`&Zhs?TSW{z<3EC%h>|iP~Wg#KaE;w;1nvkb0<7G(HcN+tg z?V7UhhLwTHA6YnmkSmep*(UtFHy!|1mi#RZn}qz-WeqGaC|;Ws4q5bby9WLvndyWc zV?Ge2G<8r839kBG9zKyj187`7Vm-dtn*KOq(jMEVD|WiibYBdj*a0%~&rat5W|oeVh0OP_ zVS$;YzIfnw`29bU83BHb|B=i@vBXiuaJxO)j&L7u>wA5w{{{4?mn0pKOiFin6GlA( zw(R%;$1|TZmxG_Zar$9+VX9)%RW@O)QBonhY?1{u)+pj>yAY-Ulr&E)bPc1>xC?CL z-hM@Tl;aF75|RsbQZ-aulDM;jbdNk_MVJ~&ZbklNQf%VeCxCWr1KodoL5$o|#reR@ z5>RS;3LG1lkUsr2CjBLudAv;?k{s{aAu0E~))(D|3k6Vl0W(Vm-gEn3nWf6AQV@i} zaA!V;vBzBS^N#CW@WJbOFH{w@&#NvOhAv9L?ys@!0haO7XuNJ76Fr6G7kjpfE}{s_ z)xjfEpkhy_o+{4rs?z?F+?VzekTe*iZjTrw*(F7I_XL>n^>Zt20ldvV63hU+UVh;W zck^HUSO!ly)*h?N?Dc-v1F|)BAjGgidRHXnBMZbj|6yw^(5&@YwU}Ax&;N(55m!%S zikki(wk9{>4-HD}sMn6iPpU|k+a7_s_OVDY{gmWKhLO!c)~?(G?{1zlS%sApwkiuW zPAuE+W;8e^GD)a-_XMno5s5qA@fK;J9mPv+LSg=R@lUbVInG$_UYF<<9_kn%n^kM_ zW0Po!OsFMvX`b@B**ca)(?di$5{VN(!zGIi*^U|u&th0r>9ke*5c#gmeU-@#L(^vI z3ZD~za1E7Yg#r++*^a#h9!u1V@=SAenu66z`%agV70q~)`zujNyWWLxgjQ7PjNvLV z!zkGB<*Cs{l#AknI&)Y!pqy~EJ8OyEv z){hzDT9Uo;&IVo>@SbqiwPhnxuNR>+zF=ZR`GJ%Q}}Uj-*>r<3>@MrUgso6^5a{lV$zgw?ro^iA%XC=H@J9yy3Fnn25p?5qAmP)@hsL)QWg9qlL2I8JTr*xNoI#jTV`teAU% z_{0ny+A+$RmrS#fdFd*n!)D3>HWA7X4A=q#@R z*}+PgrIu|1rTh@NC6B#@eC`hvC8D>oVw(w(p&p3z3-W7tkJXoh^+~W3i>Usgr@=2|SZSmvdlI+< z9`XPyZ@GLPw;Z)6JL70qUQQ;u1Y#GO41}}av1Ay7vd(!-$V}+sP&G2#)l67@QI>iH zVlyOJSYT_pxL>e=fQ=LkB62^{Q4HtxsYcuwT^`Np^sQw%!%~eZjs37cVxD~)7pXnh zajOK(X|~ND+DymDTSN*&4ul04@xtU#Y)q_0BF+~cDH^){93>ARVFQL{G8Do>cPE`G zWJF+Pb{hZOV96%F7h6KEG6#X>ZUSl_$iTG`I_N1vlpS&C-lK{;fKj0O-3BsSDN;?? z+vp%rEm(EoMCl*Gw?=v;q8Y_pW?*iq1eja0jhHK1>Dwe@{#L_ol)x03*D98Ka|k4C zwzO)T7J2N6RfixYln+;)u$z~?uTSq9>rMo?(U<BENDtb7!Dk-3J`d*vm=l*pA>siMKyi7_wj(B$J5 z!rbEl{^SWrK~57oIdG;HyZnPBVOfTkI(-OYI8HIR1VTB@iCaL}1?1nze%xya;1 z+^bNRm$-0%+Xc`x)1Jr${3O6wd&x6}iwQpPU$89Wg$%n+-c{5xGzjwLg) zsp~zzY3f&Q=~wv3e{xIC3{pydlui*og3mK2|GenO9im-b^Y{fviP^1krHdEg{tSf& z$=mv?(6(;f6F=9IZJv`2Mvo%2fhsSHiG zhxIf-0z=_A>PqFZseh#Rm0OYn=9Z@4q@cYtS=oJ={&m7}@2nJQ#AxUH8vso+G?dIFortYzD$D_7pVi6#jfc?kCkrv11H+6xzhks0 z5V7F^NwI-TJTI7n9)QK6Pt;i=+p;nuuc;&5@)N$o5{jH9JU67BC28kPc8kdNo@o;r znT(?5dczX&nHnn~kwH|<25CU@S)OY+7?}HxB_!FH9yqM7(e|T*AmI*UrNX(bYd$jA zGFP`R1W^vyTWSIJmJ}I?D1f~sM}&c}w(KG5;CsxZ-^t$?ho!{Cpsy;;t>o;+UP@;J z%@7Wz_^qjx=}K9tK!A62nElld?X=b-PZ3hv4B(yVQ<06_3ab zwNsT0vKj7oF#MieF=3G+X_o?J&@yc8yDzm&xgHd)3;{>S-geU@kpBPJd&{P}qP5Wy zCwOpocM0z9!QI{69RdUh?iM7tySux)yE|+M4!d{G+qZ7_$F8pKPvFP{)e%lLY?F+Z*&}ex!?LY`Rg2^7vaY+PQ9GyY=%I2c}VUqO2Z}hjXsKZL@*GG z)a@&h(K7GI!0!D?kiJCyrY#tp+V4BKe)UbEmCjQ=Q^r$}YUatd;Gb#+HAm&2Y6gk< zd2t`DZ*4xK|HX%q)&>*m!d!GL=nbHk1!8P+==EW5gbnf;7%KlUHpXH9i?PW8F*X&U z|C_OihM0bB`}oJ$hzE>{z;tW!AA=YhZo-&Di6qC4s^M6)@v&kem{h6nNaP9%RsO61 z@t93`l0MJ`LJ8uNdzaX#dc_%8lc0*FxanHgEF|j~oPA8cpG%brvj;BVyS6LI|1dVD zXzTqPF>~kc_?H9{7BfTsvZrgSna7OAesFR3S}b>olBExsW0VZ${NBw20!C=ltyadT z*rIHi8mg^_2ks}=u7~*$(qLXyh~_4g>0sXMnBJY$?jXnp>pzeU=g+{EIu@XBCA=m0 zevMQPU)-NeOtpd=$^Vhf#M$8eg-T)CM;M|Op(ULr>$Yh0Qo3T2Q!h@OaW+Wsx@OyiOL+Egn03xy!F-BW|G0 z9|_MSEBS&6=m}7AY2c!ynFQ{aK@^{Ix^ozqK}~Q?Z$>&!Npat($cDspmE8h?RsYZF zPLR>-d?8~tT_U-zHtW}al1n=S-&|-^wqF-f zY1VRfIlofTpt^<(mp*;E+oZ~vc0(wc1iMBgsg%VB`uu;%B^9=JvDtZrKVu11MT>*4 zcoL(3sJ%(mov&1uzR=@)$O|jn>EJCWCpu^7b9u;cP$HQ&rVLQUHzOrbs=%!eH+>+7 z{>L`6vzFh|EA!7bqflu$jDg4mqnoG`U*rzPXg977iPM_mqgGPb+gWt00GGcqoea>ytQ-{tdD zEXL~}WrH;)45DoE!q;1c@q_9>$t7#71vY5Lm@;nO>4VIDB5P1`2`Xzc$o{v!UGw+W z0=ta)B1LL3_+rJ`8JB3ptP#=;vR^vhy9E)j(J$I!7v^GGCaN4|Hf-rvk9(s46Bq9C zO&l`{jP(wTwmd?rsTlVBD-FYYzBn%%^I> z-CuvoDORxMBI+P#b5AJp{r&cASHBA*?36xxRXphO=RsJ1=+2^@*w24D) zss1Z2aVFq>t>4TXZ-a6ERkkTDsS~}qILK<1xx!bT{PJVCwQEws=}bD!Xv0e3EEJ(! zIj621v2Ph-&OF*~Hvy&LytcCw+;p5`%w*XV%nUt((XO_^DY2Z!87rO`fQB8?8rRCQ zQJ-xvk6|9jokEkc$(NZ*X{hiWWnv^+kMtj8^P?rG76jQygCHA}SUFzR<4_r`Ti4hp zc!S_C4ihTn1Kh?EeYKOu1tHNme)8YUUox@k9+MI*Jf6{_VaLt>)H2iN;S!``i#gD*7y4TlBJkZndoz;kyApU}8Y?4YoizW=SVtjFujK!|xa2Bh$@M?Q zC8%pToM8X#LT8Lr)=48A^+q#Ah<)#pWScn6tH~~yTumf9^b!c9=UpB!rmsE%xGu`K zmQDeI&r?r8p_pOzoZ`mIE~vO{ZyBPKcdQ{G|He+u3(!$$!@Csd$1EhxK~}%z9V%{& zS^VC5%R66CZD=68 zgz^vJVr~P*6nOsS;0y#lJ-sSkycoWHbp4}Yi2p;w(1!g#pkZ+SOT+w^hWRfI^Zz9c zQ}tgO=D#$||K~J}42Xt#^ZQ>kOnvb#^y5y+8{gmenvRZ-57@UG|94)tnHX4i-a+U; zUBNj8Uy`$}My(sva>vlPc4~71>A~1gx?UV>!~$gMxqB=QF;o`-eGfTafU1NnAqX$RpZb^HZ0<-TZgDvmipx%-iQUs7QO*iGgD9ry)D2&{HQJDXtF#m5*82SyM z%_mJIf_2(4Oc+xHaND<8vjuqd>4>6}UhKGK^E;jYPB0eEr zM-rVaaJ4TI0Tf));K81RtC!1&8naO&1G-ijX(-(~GLIk9$(;Q>3$XSYpwpE5ygYp_ z+B$tcO$ks^Xn=hDR#4=mmR0N~Oq6?D0=stUEPx<-@Ie^IhdZnETYv5(IL?;!pJav% zU6AqVC5Hc*ed@_};-6#&BRR~N?hqx3$+5T+Vx51uWVe7`%%|Nf(G0S_O4OTbZv=d8 zXJcdQ{tGTEzv(hmb)oUQYU;ECfIw7x_DF7q@7M{e7sS6va#z_44(Yj#nbOru3ZDYdPkqKD87kE(&R1VnqAr?&-KQ?jz7>nwZ)tqC{dEVyX+2)+1xCo4RhHmg17SX;$LpbH#&d;P@jJk^xUs5on23#FdZG>M8Fd( zK0XMj95|4G!SouPt+0B3Z5G>-icideC}dTB_p}uasEE}SVK@^#B}td9Vi2s$@6fsV zeNHb`!cpIcG!i?;GJN3HYad+%?TLN~nqFT5ge@BDj_S;jF}CUmPE6$6wkyvJE!(w% zX{Q;D1v(Eag=w^G6DKCB_fRZ7OTCM;H!`z0YW8$-YJe0or2WTUnZ;hPNvXK+g^%<9 zQOrQ=g)q!p^F!R^@7#0eFNGx05(Vt%no@q~8 z{GeTS*M4xv@bLbIAUe@%8)xNV?%;kHY?6!r4w+9AKy&{q<1pALfHu#@o_~aavJ$?6 z7C|Z7nYuct)mcMH%1&4>S`FhPYlag_X{;qP{=^%qAgL5iX`R8~SX)9!gGygm|DJQ^ zxoMj>dP#2fDE{;D(IZ>IwtZoscK7%r%n@Tb%X`f8C47sU76~JdA9K3Qlw;eZ(iRv&M)T1FptI=gTk=0*FSW#$Ak5OikOva{3;1a{c zT*I<|=D4XpX^|$ZvdpkSiWzR@sEq$8W>`w`)3SGh`hT&L#0hrs^Lu+f4$NH)G_=4v zA`5Zhi_Y#8C;~m6FE&Aj8Ss4($l^Jj9=?yWlccn8W+MCOLh5dhx-PTb=B-SFfr?|Y`hgT71B?{%Z0`zxnn)= zQR(5t{o;eZ5O}gBCt^u)G7_%}@fY8e^d;`uSPPu`hS(6_XCGj&c;?><3B@i2Da zp4ct@VZ!f2y zDFONWLaoOSjg!q%&AQk~Fr)jZ7xEk&MFOioj10EaX-sg5^~K-VB<%l9{a`kYvW%`(JJ;)&kH;b59osa1-9{64(4EnPK3wdrRiwIhh}D z(0ti8_(Z>y^5a(gsz(7j;aoZpdVFfh6&MYCy6ez=56=#(#)X-eRz1Oh{kx8k-&x4x z_567LpWIUL7K8N2GxYx@x1^YhHSjHwpZ9Qf2S6H;45?2Gt)=&_Kuv}|Ic+y^Ll%}f;wDL_o2}a`7UwSMl1?6-oVfJQ6_xhiC}9s8 zl`d|Lb{jTo#Qao6nQT^kwwp7#FUA^8O}cMhcSsm5*2{$G0-d`iZHk=b@4#AqTjHr= zk$;jIya@#4fkbj0ntGC#DWAazA@OvL3KCL;TPAWThQ0D!s>*zgqEX8C`!uX?@=%#B zVU>w2)MI7&GsSg!saS$U0frwf_3yBitd&-pi=H=<=AFU_t-_xdjyGKUflp6F0HLjk z8gk^as7Sj)c5-~{976?!{h4nluY+8dHiANeeu1t@b|KNLKQ<1?OAD-*R(i4HA21AG z?Gp`WQeXVk_V9%VDL2ZCot^wB6tFEC9sMnFV>JWjD477JrE=jly39O6{Jc(`_pYl_|DZ!U?03U%w`PG@y3zs8Q5?I`!>_{ z-VFsb9ID=7Wj3mM=YM838*I7y>3=hjir%u<7i7GB_sL!b>-@`T6W6Es70(N)7?|tC+}~zGr+_*A)E!bL_ITa8O6n=FWf`l{A(qI%t7}KGv?DUpAn+@D=9rclmjeU${xOo3gr{$$<;o|wDr3A~`xh<#$E$_Sbto-UEzgHj`b1QKEp z^kwgTz%iFPnuXWPBv~XeZ5v!{Vj;Xasnh2bjpD*Po?PKVbb~JEZTsPhpDf$?q$UK- z&w4X1Lo3tL@$sLXq?k!yL&NvhLy76Ki=-@;Au!lKVw3nv)G>1DSK$mT4r-&VB~9B# zIZqBC<UW>l7aORbgEOY$Oqa=kZ@`OjFl^gtupc4!urZ%4)nx-|E z$TsVdJ7G3=f)F<0DKA(R9Yi0UWo||eXGJpJo^%hYE!}^g@wH^o>qdp%p)FfosI&0S zGw}hzV?*=aj&}bvGns#nU545W?a)Z&pmJ8u;&a_4-Zu+@|3*XKG-vp#`s?>=k}B_f zqB@19P04E554A$EtqLzE*_?^$43aHT8R?djJ`K9a;yM6v)vaFk`$g8$){?TkV>MRM za-m6sy$8F`G`F3QlA-}InZ-+>D3iEeV)1?0;~^XdDRh0D?D(*3SIXkEfQsnVC?lJB zqc{@oht_Oqzq;zIOL7|ep@@EI83jw}J(*G8NtNc3s}c$T6aKv2HNgD)Gi4$vPh_Hj z^B7BZS(rLc(aDb|L*Pqb*UeZGwc~vxat{ApH?`FAk@8~B5#Ub#N8bZm>sYmzr06h! z=10meO9{>cY~`zqgY*mv@`BsQq@-L>aH%kP7Yy~?Xxj_2CiZW{B%myRn;|##T;)oq z2%v#@dixR<^$if_8t^;$&-55fS3YhD&5aS7#RAKBOA1-*bmt!;NG?e=1%>@=+q5az z&%`_|B9eME18hBoxD>~E7RAWFn14%}Qe0cr8<_g!8J*|=zI}FWzN`4Sh6Ht-)YGaX)OurQ~-_BP*QgNX|D1W zEHYJP${M-?CpIq0YucExD|x-RT~yb~`Cs%f&L$n4bbe|Rq_Pf}?iZcnE;HoV==ed@ zMHh?%1_v(y^9=`5R$*=<@uU*Q;{zh= zOXW#+vo_-g;`R!qY1{4`q|<;A&w(qS2zXiazYelCzj8OLS838VAk$KC3!kM9A4*0j zl8iaygeB;T_OY#HU_mNdNbRUv$aH2=>xWxKV~c)?WIM^H`0GiQw)zxu6P?q&l888B zW%;yr?@FLjaNH&?+XePLn)~?yri=8XJCI5GBdL^^J#DdOFqu)xrln??r+4pn?#&X% zrj*YBJMkkGsU>!orZ@Y~Ub0wcW`;x~G*7z`(UlMLYxkk}fujd_=cUe?(AqQ()dSe1 z?v8lK0D;J*R%rMyAk|E{OFwn#sA=&NNHr4*`%g7fqx3n_WX>$}@Vhq!K~!wQ?+Z4I zAUSMn&3%Fg1$sZk$^QE?`iL|rxnzW{G}cTqMl>ExQu$|&Y)t+8eifC*K0M$42lYcN zYT@&HA5k-aA+bZuJ8g9ABpTnWvok%oA&zx%B$|M(|M2S&F3H=hB_X366T$Z%sLAY6 zv|4^Yix3JgPC{H5-TWn^D#XTM)!KeZ@Hb(_FGf>$5A6eGAJH0u5yw5QmTc939E`GhxJ)YF6r35LBTK zb!lL4s4sQBythP^g`ESlsB+~;6bwXORi~Kff{7Py!p$Yo8~SjaikjwF4vrl)_3PpM z=b9njqy6WanZFHIWMo3kV$7z^s#!_Q*AS^Sph`w;Wu6&o12vc0=G||R4=Km{`(;7R zB|?k;)m%FN*Ibh2{@=|dn%&CWjepH0sl#vrvt8dIQv*Eh*o8T11NN8Z4Vz1vyezwO zV0#21dOW(Az*`wZONoYP8z2r6i0T!~5}7%pB)HDTmJ2NYd$O!yaF za#VfxBt*FH87X(dBKh<=Y`gygI#cF&b!Ze}3wa8jjtG36&FvEfP3Lkmhx*8Vq{H^`u=$I4u=KtE?>)C^aIUE)KhA{Nd(rtZSvA7)-P&U^&vFL zp#GQ2f2vDu+Ii2?S}KSZt)wS6LW>dqs!KKrh!3FZl22(2LL)eiTx3g*1uLWK?0kA; z!2y3Y?R5eciJL+{L#J@Fy+_>|=U!~RxonzXHX?1hpoq`}E1MHhxLp*5&C!oIG9s2^ zOQfnaB;#M-mkP_rjG8yX|LJD-{zo@sb)#|^d=5jQVUs#y1T#e5=#LnOJ71h-;YlPb zhF-=HE}B~?|BK=~)i0-xT5(>}U)GN&5cbC|BB$>=Cav$3cJ|^cDxZ0&_24t3tq{9G zTa%OQEJ3-UCJsdOW6J(Le^lY<0uhliYa23i44bdjF76p@Qyz%tSL!J93bM(1ma9db zgvb0#(&H9o{5w9bntzU89iySTDgLLrByJ-->S9ZGf;UK6>_U!f&FZ3xKJv%f;HFgN z7#8o~kqx`Ah+IF|zJR9)ovBpPd$^npN*6&qjkW%Vl;C9f0000JO`Aj!r5y(z|DB|{ zwr{#$raXU88)T7^(?i@B-0!!W@bHmQP5#(Z%U5%`vC>KPZN|X??)}c!+?hU-T?AU%V7L)+ zknFCL`XJn-zv+^)uz*>b*|C)_T^yT)weC%hjh z3Oz)?hLs)PTVPLIZc3u*2tt(TM;qbgCA2^DvLGd-OHj9E>lP0e%VNJt->DhffuAUB zwd>Rw0==M6Qtnfy9B;?;1P(zje(w@_jV;NZ`!5^TJ=pk3$`wjCG^(>85!3 zY)38fM?)bzJqkYxI*%xxrThvvi8GP@JlaCr#d|d5S>@_)3iFIK_UmbpRq{gB`?a7P zX-tFo3+L&^)(jTF8G3?tdUdz4Al&UDpOT$&p%D@@%%Bv^nQth@0(;6)ah#;%{dsF4 zVuRh#7_|9P&~x`mUWS*jLpu?va?bvil-VNM3TUPCZa^|9Xs51R$B~!RnPp#Av1lqn z{|$lK8`yVIN+2%SdE-*qZWVv~qeO-cjI+Qxlat8jl!G6vylY}Cy%-v&zn|_$uU(~| zG(h%eIIFYiZ?-c71I3py2ry4DY{tyq5@X5!dc?isTN$e7WMaSL7D!1>rXF*BjVYD( zHVKzEeD@HDn$0idJR|Jc5h%5CuI$&FMTGPNrG86}{tb|h02j(zre03zlTfta>5W!F zma?+dDr%d!TN!jL#ckvnY}9O&BdfWo>z9U;HzJF@pA1U-L%R2c7w4Pm@jv7#`#vtc z2~}fc8!8GXzs-i?`fj=jW9aIi7Mh$59M4_CgL&}g+Yta^Ndx(~d3c{EE(XBsNoN-L zG7IPwO4cQnwZohC)y-3FF?4PvoaM6_W;9_8?#jgsqVRnaM#zM4tiG7wMEpraW2sa# z%!X#`W6MdBX^s}F9A&Veej>k#ne*%2V{tMP1xF-86iXF{Ri)AY`B3JqOX6RV+=USx zIl~`aT|lr2b9j=pf}?73`o}KNr{YnMgk*Hm3?ecsF7|l3FjWEc?~)02_-(2QkpQKf z4x^ns1sgy42VB&%e-5+spG*yqMcs9JNujm_+pgg4gQX`Q>zY(JP<5hZiZ6*0rq%@f zXUafZ(&~_SWitB6Z;A>lZ%i@~r{(e5KR1bRQzc754=ow2kPvgiRQgNvd_1$jWV%*7 z9>vgg2aqWui@l0A-LJCSW*a4%mJgb+_rfeut=|rMT1MY^4Qz;H?3-8~)880*D-=7= zBbs~(3Dc-;`6pJAjdI(%>#Az8t%9oIw5~{N!~7N55H4SV96s(bdcE1X29apVbPtkW z$TDoUcVnCCe0|g3>b`vQnUY|F6gah}y;RB6S2Km=yNU`xp#g*(aTq!AnJNGEG+Yg) zw@2SX=9xBa^FP#Cz?UQwJ<1@3%1(?8X7(azyjm3|N3e>xwRJ`8*RG&Bi%6xkq1P77 zJN`A@80R6=ChsRPfh+UU{FRA3ErX_YGhUh&-8;&Lz z0|5v+=zIMD_E7aGqT2Fb&$e-6bit-?KEpG2=6GmRvmE*l_hs5bvc&xAS6R2_=?NO%!O9bf2z{R z+YC*ql43k8*=UwH_hsUz@fr8DDWP=Gh{!CNoB!7?oReDbR?xR^WiB#4nPKp((A2=? z&e1CxoRPr_J7TcW57*hMjv#dF{wn64sXQzAMpm6l4eI(c73EH8pKsnE$fN^9!l+{# za4e!9tGUnDe)(kfs=MZ>ikFgKH`CTg@lof*UG8le>hhR(T*160sa_J8pwd)J5tiXW zbCd>q6(T)nj5O$E@ zTLgLZkc9B_xo%3q5j-oE`ZKph@~}z+c^5J1rDR=p{L&g?HDz|9m{tR!@lTN}oui1S zpQ_Oy0dgLStb}QSHWve0_lif(BX7hVW%k(CeAl_>OKRm#aJ(Q)U+?{-AhnaSNG!wm zBG5O0h}%KgL`>(iD%#b9vYNUQW>;>TMi+O%nmZpWwe?_%de|#q|GVd+{{iSx_WU~1 zJ?!KRHXAq%+h(dq4M)*_oTl$1Vn%@-I2h-qgY5A-^8^8aC(B8X3`$f91yDLoR=UFt zcgGHL#D~sGW*aUmg{z({jg^VbWc&ZlxKgX?!+A;b7ybuEBrsA{B?o!DgM(tX{7H2M!<)6w$ zB1j62BeA8G*#!7KU3*Cp27ATBgxNirs$=rOWEiA)lV)#3vx^mwm0${v(%b`g{AhB^mWyQ?+mGi^C(YkV10c1vJKG?IJUo_59kh&6DwPH^>jLD8d zX#WJ?L`C#YOdOJpJs7+(68F;`h1Y^*v7j0Jl!2d+A8yM9RUh}|@8=tK0DHJ!OwVKM zI{EX7|Mq68E*&OjCX-3}gvIHmSP%k+)qXRIyij!fk&jY!sQR58+Fb}!QQVGp`4@q< z+6Ir_*{e#%kdkYXQPEvje<^moDnD0++*|6N2{3K6CD~=FGox6lX7qKVpm%&ly5+a8&)M-xf{qb zy~atwwbPDaFSYbiAQn*CFH4jE14b0yY{@y}o`7jz z2S_;8)qRpO@Rbh9J&2AwKqM~k$M4fILu7;Vqf@x4W(ofqNZECr&~7E?9|tX?a_8Dh z!$&P82TBxILCw&}KMA_#J^d9iNwA6CN5M_UBqT*$l#dr63mSi&+jZy7 z@%wdboTtnikFp1+T*PYd{l)%T?FUcrhzl`ihzEf}_wgO(k{y%KSSN z5{(Sz+atDTH@MiDz*om`I`EuP-4X04uf?N_Yzbr8iOlhspXBRR0H!Wk zvH-=9e3RDvayItKLpf5LQfEzq@<&o7Ag`4dt=r_Trg;Zf<65bfS)0Zd&5HZNXlzNS zY8|5XCJq&Hzhd}{lwvP7olGCmVVk@`m%CNM+zkKz%$ zwOck~{WWziy_^g8;x~_3yR#r%62sS=&HOnttU+9a+l`>_QU!+vE2M2^X2nACSSOh1 z2zaSaLC<{MOQFNK8_h34>OXmKQHzEtNO5Pv%=1+)T>^|SqaVN2X_`+bZ@}l+JlYC| zfx}VXeX7oUqvsVX;)HKb1^50vfq0VB6@`+=9vAcBoN?HDu@KK%YJxacVnTIPflPb; zU5RI_j)8)gf{KAzSQGj(4zT5>WFT$G6l>X1ku?|{>3bWv3{9VHbqXn16Zd%)&u`IF zvcvE20564GA@O!2!IIh}vI#Ff!ESZy#IY%ekLKX`#_b#PK(WLPPjg zv<78?QJ$a~7AE`yI2ZC3rEHIaLx`*aC&*tBfFSY;!{yDr?g1{xeo^wZU^v&;1^*GX zv-8Iqa>1_yK_NdF=uDJyll|;hT*0RGY2*j>g$dG8hlV7OZ`%$@w9yu57@*IGAIvmm zvO>4G+taN|q#xSRni}dPW^7nbVHHF4h2c(!svuT>{m7has*}W%E_L5Lk~S^-so*ao zJdQa4wpvz&aFS@HyXva^ktw(STs|3fD@wDpbBx=y=hZc#P89mPhoZEjUoWmHfjEN} zz1B}ERZS*++}0>z#0GsMMj3VfEPiDSbE=g=Gx_D~vNBx`!98r2K4^wKJN03!wa_lB zqQfzuIhKU}_;E!{cwl z&cb3(HYqU?%7NR-QcgMp8!1uHrUwYr$}tA{Gy9;62_mL@k6k<-!4C{h zrD!@UeTJ;i$;DoS_>XUR=yHXG9+LsYHQ8u*xquWIGC7|6u(X|e2Q7@hChKyHJRmi^ zUxT2(O*L~9IV?Ov7l8mOive;p{gyqohT;kNT|b@lLM;IMbdI%UyA-V6EZlXJE~#c8aF)IrH3nOUJfh^92jh9WbltMItdyx3!GYEu73{q4y%N#>(HGyvF^EUtkS+t)U$u zIFZe3wCeSRh>niFs`x@5T2eWEVfhUqu0O}}ST)BL6gAwk@SR;9eW$jM>oO4X#t=L< zCUM$C#0;l8$i`WKA@X!!CZVs`m_7hweabXC;W^OH$R z<*UVBN4=A9`JUSO^dYIOH=GXe*#tV03UsrS<`zol>~$?2>7}3SdbV@J`mX)f@#_J^ zBw$XV9Mz~a?zl{=oxT&~_=C-y&Qne$$IJ{j=WWC`hL)e34}J#k+EdWl;8v|v%rNSn zs2a7I-2;rhl^=JIW}YTUTw_55=0g~XRQ5rlOCx^^({s5eVif(*{8ZLDFPMcKw?*q~ zgElhXVrXXaYs-mI|ISDsj8V~T4aMg7B*oZh%;rk!eZ8kPz9X6vBqk}h9!vw4%#cL5 z{i0nu5KtUQ)Rk_>kzS8#snRWRzOp}2lgv7@X=DIAUwoq@obSe{$y|BwUQH?1O;SJe zM6o-MsxPat7Xi8g9S(p$uH7aqgY~k2?k&f_x;MKMQs6vnf4NUst9I9!uV^}{kjN5-ivw<-=3v08I0 z-=%|+VYnMGhV;b3j7_>EH(>{iFZxi)JynqHD=$4$pPHXfa`izHpN1Q6NWhvB#g_lj z7Ly;uSkM==RDaH4fG-)s69!G`LG-1E)p|jO9?#U1Urb4G>>k!}1{fvlcGx0y4yoB~ zhW)Eqpz_}XVc3e-W#10iT-j+x1Ci*aMGka7GJZzFhgAL0`sD`SnnfszCGm#4lxmCj z;JvdJQku=JvH8d0>V+m}heG1FyWQ zB5e1s-Vk$b&|D+V>8)Y1B9$=~3jeo1*GIcIJp^6;e;1fb0)?PCZm*|zBy!ph-ZN86 zOf!oEd(L4u0-2w8qJf07af?A{CK$0hgfF^-BbQH=D+EePwcWtJ=i6~aT9zC!uN2f1 z5@8)i3fq*B>t%e7;)T5xM>MOuNasnLg;{r4i@2o4ZgGOL_LX+wWDBX{W6np`@&)&8 zhc`Jb@FuuRkDn}DvMR6pzte=e{5_o>9&ZmXA9k-Qdfp#yEFA-0U#>bD4sv6+yWfgS zpoL&@a2F^|jUl1|qZ6U^67l%E)n_cy=o)jCAmmd9*O%3bjX|4a`*pLc5P2No*yc=2 z{)13c%vOTu&H5G!d};QwLfc z@bqQI>8qABo#6MFqHw};{^{2e5A4GSk1W+`YwSJw3V*Vsb0m-uX=J!VvNQgZqrf40 zyYeR3XTb-kQ5yT91#`7%fa~hpEI|~Lm%`J;36USt8?;*ZgXixTrHombD@z85IkXkg zf-+akuPKhTnt|=>m1AFu6>~=$X{_3e9#%V%hjE`4c~fGS-GiFE$~2TTI(XwPg4F4= zd=_|1ebhAQNjmb7igDeG16@6XKlWBu;W-dL*B(VDZXH{2ZF;X8@oj1ydI)WTu4{>H z;*aekhRbRODs1qj)HTYP!!~jr?xCFTeN=Qwi=(&6dYwg4>SW}i)*S};D~hG}BFl2A z&Ey#jt2CnBRsYtm8AwrF90S%=0Au@i90h$hflhQI(}W`?hxn~&tedHa{i$2A2P&&o zp_;12*l=GoxLDY(qwK5|pWC%F5kTy`}~rFYbAz5E_*VkMO+ALeZgQfenC zmh?H&=-v_>3ed@_nyBfe5^OYPBpSOvC)XFt)ftWFs(i&bPiH%u=x~dM0N=G%{qC7O zawVCoL|3s-H}iM059`fW^T*Lm<(@f+XmzK|%{&DvQbp1fB5fY&Xzkso*%gYDz`Y^! zbZmQX6(!VLCR-3Qqcj{R!dvoeRd;n^MBXQd6-t_Q&_-(P>jIvu?mh8=C<9H-w6O8e zpN4OWo;k}OAo|aP_+9Awpe0$Tx|B(RL|*YN3MW7Rjjw=VJ|zSZhF1CAF*D;ih72S!%!#k6%{jonUk(d-m6%+qwIyo_*&kG+=|-6S`qp2%-gTQiPeWETC+KG~UbhwJy%`G+f5jI+kpdtLlVw6-t-#brgj( zkeHL!w2Qiy7t*weW&CZIk=nzXR!eCLN!g>2wxEeCj?+ksMBw*|3f-SIAS*HV*n~xk z-(S8~#UZ8tebu2S(|P;EDlr1>-gqO6&Q)*BedMYQIRAbVDt9$xB7|;c`Lt~z0@khU zYzMXicSde<_InV~dxeY%Kq+Q3@@G;RT7=KcW3emM&%qr;@afL=IuLadaDwt~v-Y<+Fo+yHQq7C03ajfISB6Ej1_cW( z#88@WVi;8Lr~_7oW@CBzTmnLK`Dx?$SB>SDMfP|tuh~4uJ?6sZVm%chvGZnB{Jx~AkzxFX`YeV8KxvI`>%~NqTg)AmE`?ma z9Y~1%RAU_a88J&f-l{;HQ==v*x0~ke>h7H!?BRhB0xXRpC5fh zX-kS-oZ;B#mpIb6C6#B36!6Db>$$u{-^F8?GVhzeLN2DaALq2TdnyLPMj1P)rP3_@ zZ-p?)W@Eq%V#!HKmnI0S`YC$K zPI4SM;o6t((vo$%JZm(vyt0DxI4Judnc3F}$mws-i3Q^DI*IIKR=dzO3Ns27orMPB z5&K$!zTUF?g=Pj$32Fk*%=tIa1A)l`&ozE5-KTfJQK2>U<%OUyV>cegFIYu^Xtfvp zzG#R8ayK$c-{Eh@0=))paAy&#x_&l zw9J5k&H5|AhZf@f!0VG0(h=DQ$`dkGvV^37Ywuy(GVC9lParh0W&q9h=EIF{qFsbs zIY3dgM# ziQmzRREMqE%(FB#Ol7x$(hG2$mLUd;7<(JZe#u7QU2M0V3gPc58i(T85WM4(Z712e zxl-Q4fMiiPAtujbifGe)Jm>_%HUpSoC>dlgH;FAGB+R4%0Fl7WWwHw3Q?HBP5iuY8 z1T{oxSIDwc-4T5`WOP_&pFh(D9Jkfa2_}SpU(`GkjmBY-z<|u4M~KJI-^+x}S zY7h3r$k%nz{g}k8UZHn?KvL>6+YI}+ne@VYW&3)`C!FZ~RnGLvkO zPfag?x3nOeI9(Y~ z8gRGchm4cu0EDz62Rn7ft0%f6;kAn?{`B^NY07!@88q!defqNJ4Y}JY;OZmZD>Gd! ztQe1J(xzqR-Z+}YG#YK7PCp3|}X1)XWO#X$VTyOZAC_8eaF>M^oMp&q|0$ zqM(FBA1DFf1>uugvIUUSy|1$e&J+ZDJE|Ided3_kA`uQ*g5R~>y-Gr{CdJ_E(=#lo zCOV;iW$t!Ej;6S$0=E>J(3YOo0!M#OOi+E)#r<_-lhoGK$H1rgmhB%4&sGWMa@%7_ zI0AK0O0&vOe+a*5KXi`n3NXz~LRa!I6b$rkRSXLWv4mVvE_VujqI;7M33lSO6lay# z%rKEeU@OR)BxAB7`QzM4fGm7@AJ#z?lMSwU6#tyGXs`4pe0d1*XqTZ`exnnmrfkjE{&03P9qZB?KAr)@j0F`t zY%BwW3jOdZ=sS58+vIq56m2GCyevl%jFJc_zyu!}85yaEnyoCVG`yK+Z=a`(2z7b< z?|ItQ69xs>R7X?KAg7gWqBWnckA#?(ZjrEaT0*jmE0wGC5$Aj972mQD;`Ttv`>AZ9 zEkN$9Yh3@!Z?Z3MS}7h^!GYN z+qP{x>DabyJ0070&gA)TZSCyTzL?sXovPV--kdtG&wW>&A3opf`f}vW6`?ZVgTX98 zD^GApdNP&6UUegxwIL)QT426)^>q^t`RdE(y#YV)WO&Gh9pZJZ(pKalv`V?kZFYAJTqZEMM^J3RJ%v3hA#2iV?t-5>zG zX>-4SlOjr(Ai24W<6hQ5ZHX`YhXqetXk-824`?tcY$V73k+>!!xi#85&OL#C|SrRQEzfy)VayJE<_8u*?B5?_6*rU<6Y8us)>7=h^2=dO0b zlm4O+FnJDs59X+^_jaPRs28p}IFyfK` z79sRY@t&UbfZ7q1Y!e!?io++qHB0|MCriF-O{qr87=f-IC)f6!IYi;Sj5?xbqrWkYXghG<%B@&v`oPC9KdT8P{^sQ7X0gBJ2*mcDEBzgJ z2%bj|j|LTDOMzMiG(tsAh%QL}S^*hACNjY{k73+RTVR4m_j4yLe46T0y^M4cmGc%k zTrl{RV>aqTIdq*KC&K=w4Eo)5#aB$<6?ZS@LK{k>;P3Vud3l67kEa3^h75*ZSynk>gGxzDfuK+Q*+bNhYC{v1aqicWL)zS`$tL^$I)3y|-a3Pnvw#xLgQU5*i6I~i zN`IYFLV>ph|DJZ3Qv3B>G<$nuM^wL33b=U@yXf`{$SFd;*DM-P51-G!j5JIgLb%B9UJ59fysBagfOw5O7 zt5O!?N$L$@W~~kgjglv*=|(qh8nhYm!V3`xYJqn4-O`DrYWXr~M_=C=k?1gxh#D31ZT3!8MCtOvS3Ar#NC~1S*Lz|d z+`GcuLCt{^c9{eR$q~^Ci(-mTfFagX2lB{)&){d~L4$t(KoPwrTT$HG$74Sx(Omrc zd1Bu%7E=%QK{cSh?t-S&NlYu}XB012uC8OJtGy?t5QG^`FEO*zDGVL#HMzWn|E zni2f2kv~#`iHgjA&{W!dDpA!#Z`7=c8Z97xiC9pKC*MEgL#*e_YEPxQD1UjhB18(* zzI@wGW<*abA*iX6W~AV0Eh$TFD>WK9U$}4rEekulGf|O0s~L5oWU$?_*6>j`Wh)Dx z_^jRFg72u-CGRcw+?4L5W|5nCXDOJk7r@Ip~xW(nq!nR#;f{c zs!vzz-6S$SOe;EJOJ>&oS|l~*PiT~*mYGLDyJP7xQKed$WBVg#z8@U7jpfHo#Xdhz z*xhTVhagavZ&@G%R#^+xKJ z0kxjEe-Quimu-9CZoj$djAs-nPD$X$QOW&tm2xPrAQk?ObKTK`$!F}z0Jyw3DG@P! zVUnjR2CT|&Tfh|Rtuet;7XE$_93lbbCnqnw;4FaRLWody7J+D2$k*fm;5lym=pgcDm>TVsJqrIs1^I5i{BIhYf@R|CMhk{(Iss3OsPn_l26g1P5yEEzn6@paSR z#8tKw;U)__S#ph7s~-@$cinv#qgj#7f%eJhpR*3pb_hGC3j@;qRQD%(qm9Xm9Ql&7 zr_fu2O_AF;2nWT&57OP8opF|meHBY85#Ust$`M1ZoG5f@z2&+lmsz0A1$=0 z%M+vu@}qzUtZ9AqK-EqQ_hz6{!mUDj6xy&5G zRl?q9)k039iGU-YrqVBf6x3=M8~jMGFXf86iGV?otY0}Bsff8W_u$?zo;~|dahOYs z@_UKP-xIQI)optK=hc0M-h=i@LKj7(Do$#*l>U+>lX(IS3%ON=Wf7W{9qt3Nivqd< zj=!0Xw+7Ewj3?&fl1K~GBqWpT#>0wc<^^J@N4+o|pRXFSjO^26QQwx}`L(wI9#+ZI z9oVy3N?)B4u*EUWOKa{^2y@5|E3zcVjmX=*!Re{CVS_wgJDL@NSSEgji`7+7g$OwB z`=fj;;Dqn5>n4bc{Ql^(e>b=jDefiPh0cPuT^p($2lQYoS6qHhcPM93q-ekQzABr$ zlEcH!as+{>qZKvsMh*~+Mi@&7kyWh!pCs` z2N&V8Im#$c1?uvh5{(~boQQEx(NSa@qkHOOXv$+h#DAh3Hr`)IjIFXP6P7hbc8(jc@SCEIx-J#+FVbTZFHbXhszTcaz}wt^4M!mY1NyaT4qMesJ?SnGlrt5t$|PRvk90GHXUFsD{Y2_-5WJi!@w z;84ia>|18%O69jxJ>^#G>y7puSDLVvf5S?u zUtI*OK#K0-Dn+V)(K!vhUz^V4GJpEc{LF>J{rJv7_~EhG98S`W>>J*cn&<5_wjV8W z3yGicmfr65!zZ^K{P%cqiTS2{s2z7=F3pl0?c1!es?}k`48fOG+XUl&dYfGAEY+K2 zP6`HId~62>asTRP6Ydp3H@tEwB~v(hsE8_6?hPX7g3l&3d&MK@3PktwTJPCf64<1L ztP9Kx^(0B{9kH}h(gbbE^$^EacNCao^pLj`t=jI_`XLwJUFirA1!snoVvh?v7V;C< z_0NNsYk4G*u_4w!<4h(cRum>Gjm<_VKM5`wo9HJygF>|v-YT8DA=|)6+bJ7Xv00ig zH%c#7+x4uW7qwfIsLJ=*DqUYV=m?#_(h>XL+zPHH*U(KRE3>viqf^c-BvUVK#n9ns z>c_hDY!GyyKPz^FP?W5f#W8cnuzoa8sJ={L80|NY!g)S%?(27}M9 zF*%5KGoKi2d#HMr&P;7W7T1pJQgbKbqer!9OQYBpa>kt?0b=j+go{VGyMgg4c6<5y z8cxE{58cwA!zEUsG)*fB)0e~C6k!K}3xs-n!> z{PasLy2adXbQ{B4ngG8ix4K1Ya{Fwd?L{bnPahv+Bdmr+nQyreDP`2O!OJ3 z6W8yYN~MTg{OvVPV5xj`nbt@@&`QkG_njKgzOP_*Qe^4Q8RnuR* z!Ynl3*oZaFi-NPZCJHIr(ynIm2_1_zosC%)M-1UO6%>GTh-JQT&$e~6m1k5bfdy69 zJ&GSZc2AQnU5IWM8KG`Z3`Zu)BDsWG7yi7i5p?2NHeA5%%TNH z;rnEUeq^wjD?8&k!H4S2NBB6Hd{a*CQT{qbOe|bJfsi)iMJ5H+k|NsE=vQJ{ZQTp5 zLS4b4n9bBj%YJY0xL@By(-f$7)7IFvHD=xaN5k!dVZ4*l%+c~jCEa{7yzgKSDtd2; znn8OLb1;M_KR^7oMAcgb)O*?x;VJ$t+V>5ESh4E4!zdDEt5eeC8grJJ^bzrmc^62W2SModa(vVNB5Hx4DJ2cdWXE8-HZNyV-OrvMS#yD5V)N z3n7aL^xW5kY7K<4R|WT7eV86B5;KGnPvkz+_qkgCAWWJiPf@tsxn*kI7orur5gR&e zhTD3A4EzM})H|8IoSWw{$~Fp47u`4x0ws~vb^1%vC28!%~<_~J)*i_Z%25NnFKvS zh_M=mm{k?q(>`B^4=+>S4|jnaio_gb(tD5p*HjW8u+?63J~3+$49z}Nlm4)oSZ5Hd z=P(!MU+H3VDu+j^BMnv!Sr}1mM~p<2D73-%*!AMwBHgi`&&^`$`u1r!*-j(HCS+9p#t@S&bI>PP? zlhtfp8y_^nU!6VFge%|L8X|C{_uZ5Kz#zsQm1d>AA}G)PG7{6aX01tGt;wsP&wKQ* z2s?|Z9qZI#na!ZG^gfA+Z4T$oHlKtx3Zz))vndj0VmWj%)KH3I5wT@&lm@72{u1d) z5$4+n5S90@XYSd%>9sK1P^x~qk0=(AC0j~#kFFW4_VKbK>Jx4g-1z~)d zx=-H)Vg0_Ev8$@D2$y=AGUFV# z6PHmm3x%|rq+6$>#C3weE&)gdyjOjF%k>2-pqAmKYRd{N4@;<8{W}-Qaz|qe%NMPN zvA|e(F=Nf_KCMa0^sQZUFeAB`C6?CgNt38uFZu4a#9&=xR>twqiul<$Y9*#CZ^HYik0e;1^4<20 z`~Bz0ZX($X-gnB=%jV^$C^ww;JBApgbha{frP&PpeAXX?-3{A2jAS_nIF|-Ooc-%w z7Y+sym%H3@7qZto<3~4<1kp(Bw;3m3!o$+?Bw#8Ut{S9xaT8OPoAtel@QjA$opQK1 z6=Wlbz2+|~q4XHsQ_Mews)NqrM7rttIj&i)@(#lLqoWWM_8UZ*Pd7_J3`>#0^UjGR zGx(UVNe?&;C3iWv1IQ-YnjMZYs@webnJ!!O>;sepDIG>NVgNMR#H*zT*o$=Y_XiD~ zaC!>|AqWu_;-0ndyrodL;-%kWrNEu%qllJ4XCsKTYE=pcn76N%kr<>v*6f1H;%W&8 zO8X_IaTDwPBLl~ozAVWAKgRh0_xjEbw1Zs=${pxeTQ)YUao?GopZ`fkG8NTPKm%Kg zR2O*KW9p(}+@o=XZy#j-u&8e3(P~*C0*OO(<@59cb!M`BWWIqA^N8SW*G|ywp}B7} z>t_XzmvE`d7@iyBbnmb$G-e`U(9F+rsu7!k{;mg0)V+=aEc;a~3uzS$yHSW{Uecx$ zzAo^%5Auj&E3!<`TOwT6Z95^WX}~3%HP0S0#LDU#h6|DZP1AZg0Cgxpc847kSN^_O zP)%Wq!_gjsWVz!K(Mqcj{Q);@n`w5~hf7PaZ>syCss|{v`V}E0LO77B5S52d6t3 zO?l=*Ws-8)X&Iv-L+uKM`c*1fkYRzo>@ALct-EX6F2H%VukY}lyHk7-d&dxV#WbLS zZ0aL7aKKddyC=)r%ViXb8al<$m~+#Bi9%|e{JSm+1CHhtY`>d`^4a0L9tV^X?aDj6e6QqlOiG9u9urYId<2vU6f1=1D?1J1-iwcKN03vFTz~%l*)Cxt<}udhwd`5IWI5=P*-H8Yn&t>k<8P^dq+ z4LJ;ji%;~s*#*;@vadyY9Zl3<8EwjZF$UbJrwV)O3c5S zVKb=%cdI?5It=~c_7ECJ>fXP&h|?{Pcqt19ROI{KT==q0tMlIrd|;DQ2&xM@u*%V?Ll76r8}!>h zImo_d6U~y#7fH6Slf~stkx*!}GLOX*L{#8=XEOCFp3;;L+$JJ@2;U?lF=f{nT|JHh z15=P5zBA95iHC5PQxq4NJrM2<7|AX_L!S%`5a`olMWwJLP7;eb8Fg?WAs%qc$3k@< z8}7c(1v~@TdE-W6d5gfXxP2}@`2H>>iK=_5jE=x?!xpFZ3A#wL=Vz7hgC>Q$3fCh< zn4z2-uBDR`D0-00{=QP{Log7r^Jn`oTl4TPfqG1$npZHijZbM)DYsTCZ!`cByp3K$W{`J9>eW02=6UT)00U|7`_bvr7sLeKz|_g42UTJo^TtH`h8^2@bTzGlp=4FU_VzluJKlJyvSX0Qr0Alh(OlV_1VEWE4nNiT3tO9+*DWyN}yvwx==$&lu9ILdxZ<+*3nBbP;1leI`sI zHVoEFzZ0>PXm!8=Mo7|KGBIef2kIs7?I!wC_zQg;l2r;d3gcbzO=7;c{RF2>$=YG>Wnw&;07&vW_x<&b{KY~Gx^dLYW`k*qp4y- z5`(~crh}#Q<^Is99X}?aS(roLqACt?gH2Ec8b~;tcLKrQ!5Hk_HklG-{PbU_plE0s zvgm{g_{V%W{u<3-;!n}zko}CPl?hspq<<|8Q7Y{Sd3^d#(8D8YdX`hGh$0tHXXi*P z?!orKF=(eGAfM<{+9$@aqSJoq*uRbPly}4&wjsnw=z+3J3V`G4o}K}pdPof8 z>8SIYWs#!g&Gt=!2P}2RH3O4^4u)afkgMk*`Dn-!{H%N$fOd~ExvNyU7w*Uk= z8Q?!C57ZG&dWps0^LlV&?1;mFR-hIc6J9mORvwJ%GGEqB;FR_!Jz03$P`78i#pwpK zx6(N#wQr?{rw3Z8O%)HRw0WCsRL$boitZ9(w4t>SCa(Z%&Ld9(uG>~{9O((2M z7^!cooFN#-V+un9h4}n%e$?;Sf$CZqfpuhknSfMILJ2NHBSB#9Ogey2iZVQYXQ?<_ z@b0og?1n2@AcC?PCQ6|i`0DM?Ju78044;qGKb2r?DSzTo=(;i~gcY$nK9|E2hR2$5 zz())9L*iLYjsK|xlfDGUACrm3kc>HQXD2u)uevFy^6kzUZ1hF52I!6sYJXtT& zx!^dVboV6qPbJu(fWUb_GtAt%Gu8=}u*rx*Zh{L-_aE=bR{ULcObODQ2B2si7~y`2 zVMSNYlOK-l2nE6Vy8qn7k&wTZUTu2ETB!j+0WC?CBT@E&msb3i9I&2arW@5HHnh20 z-1zmi1+M?Kk_T=1W6w4;hqES6&8QN1`zMQ7NL)8lHVf4mBBr(x{1jOqIl(m{f3uhx zmfYUFLYrlu=NPKcpFS{+YO0gSscNRWG|eiHh5eTHB-`v+Damwy;4r#~S+{y_s*@Yy zB(#+I?4g#rVJLZfDO1zy$8#H`Nj1B+fJtLa4kcbn_=KbrN_(=ra6L$bM~YYI&G2}8 z%kLbdN^K*p2^MPb=%Cqj6GUVM@6I^X20#1n3#G|_a*%>?bK+PT4zn>)8wbYvf9ZHz z|C57MD8QJ1>@1eAFwr1B^@v@#UQe7t0hZ>$qDat63F4FFr2p*tLxB|QGC-b;`7;Dx zWle;WUL1hXT1hthtCvN=!oWK^${|hschQ@62Lm&(@j&?P7)JRYf=rR{xV4I5mNHUP zCLvvRY~aIQ1RqN$j!Z&555PEN>5*iF z$3)hx7Q_x{pG*Sxh?q=Y(so0lc)!;9bUJL(%!L_xA^uvg=0DNQLg>XMnQ4;}Oc;Jo z0EHALGFhQtA)?_sO`8kQ##v`7KD4uJDl|)N%ul=q`JW?SGyoHV9AO;36VE3>&_}rS zrEO`b)Vx7jVwbb1NP{z;=)_Ra7N-eiOc`q^jGf zC`vl)+RWf|aPS?)!nDwml!H=t9lBD`XxPGt(8QiYLaY@E+u zr_Lx8plC@Pt{Jc7nTwHqk^FE;8UlRMqJCfDd|R2(G93N3i7}L) z;Z}+EVW@FBMt`-;NeIFs_YNx$y9Kdw}bGkTMaYX3o$!KU5|eTBSv`?;TyQj_6>p5#cf^ zf{o)JB+rYln+Vv?KtbOfxY1{$6JCr6-z$xdVlN^5nw;ATV)5ICozR^Y>@pydZz=W( z5o0~u5W2Q=f%7MPeP9!(sstfAh=ggPoEPG2*7t0T20}(H_;1u~Ahaz`|GH*qMxgJZ zCB%>VPCV>=8i-$%u-6=r?d}@c2Jv->f5pt+oF1-VKR|rVZjP8|dh8!$fecasKoZj3VPVDwgq(TJZrzuFBCjRigrDF5qF{Taqw5ZO07QFtv zN#AcQV5G-Z{l|W-X?|~lFX9!VE9aGw1F|FyPLxdQun||RO)hO7oL#PT-|bwovTcj} z**INi`GKz9mjw03J(q;_hT+ZoUPjjla@1^fNTF!Z3D=hLH%E5v#!E%nX^$09Seq4V z1HRY#bT<$s2Katkk=!CLk!`( zT(a>^U&qTJ`*R94W5X%{RU^Rr@qH>&t0w5$UA5&_0y1!p5Hq=*Ot0#cIln~W^KP5v zHzw(exh`e6TTvJZC6Xqz<~cQC9jb;}`&Ui%dNHe7(VLO4ppn1c1?&?)DO3>t=FDYH zw*kThThnzi`{W-K6mGKc<5wKS3>ZWivzFWS!@44~WLkYTpw{UB`Twep%w-LV^oUGW)DNKF+NSF(uOhiLQ<|cVLRacbV~_z^Ms(FkFCoz1iE+@8LI3{tJtVuJ zQpwFwJ6=wq+m}&OUp}h`%4OQQ^Dg$df@pd-rm7TI6lGYN5^yXgA;d{(h*xaxBYlrh z6g;k{&FI&|iMZw5NbwDd0(C*BhM@aB1`Ec%lRBzESOb36RvmPNgoPo}y=bhV8l9bZ zN@y`~9ypI%#IQf2qhb2%Yx44{27bzB*0y>vbqU#Xi#`8I^Dknb4l-mJFgXw0rdCT1 zxoA=Nu;EY!`vyZm-|50ROy+DFD^cRvRG`XLF;dRUU4Id=C!Usp6DBKj3^MJHqbvgv zG(i&;&|OA<7ZRjK{$Nl6<^~lP5oLOc9q8BbW|*NQ?(v7VDwXK)ECfc3@W| zig7$}3R5|=l-xWH+oV8nWOgx%Qb|&ht?nlAp@Vb+3$ba+(kR^+r{=4Ok07j!z(zl? zR-*N@c(j2#+$%IVVKD@f44BV4{D=A6T;WflDfcP2N|pQ3<3tJ@8$Y_BhhVgV?~n7L z{a>cX@FO}xIAYyB?fg;GctS{yhv*f(sX2oVu;OYPX*g=kV;;D(1gOU z7uOg;fe!Ho%)9TG;VrM7_Zl~xSZ;39fdE}yngKKdrF^UfJ@EiN>fH6GRD$AX^_;ld zf6B#)|gMW(H-y*-Q~c zJE|6GTD%f?q{5+tUxi-+zRZMNHdJ4iqYE6Zo1+U@U~W;qYmSHQ&?kX)zk*z55}1(j z1S*+%7>s+SP6K7iHQ=YbZRtWlj_3eo253P3-h>S)-mv~OvSzI!r39^n8$lY&0q`99 zL-MkaKGo+A2D7CIzm0r|tD$uhem#YW4A?GIGH)B7B<(=_5H)86!RU@I5itd|gY49MFCbd&ZUC4tAT(PJ*LJmii=SqPkCJyTh{x@)mY#L#?7F=T08sISCIZvFF@3(eyGPuP8`Q`QhA^s;B5M^3VJa zeTm;AXjh%a0}O;V;ZM^+`Mjzsk%;!XU#p7hG)Gh*(3LNx4P)mv+_I(_335BO;Xij3 zZFg8GXwgvyK6-?L9nQn;Nh|Ywwk#BfK7iE9YX{~nZ& zp2GeQd};QOz-0r*l*&KcPMWISEmvXgpUqxxqmTMOz)L}-eesy5dYc$ilDwW8=2%v+ z*QZ6hYSk-e)LXVhORpUj@sTsWLJVw6ow}zoNt~ndyu>lE9%33Tl{w#hr+&&Ta-NjA z5oMVUOKY4*o$<6B+{VI#NApD965*ld)-$}=86KZie z3dSXKMx%s3e_($zhXPqK_%T1PK3Q-&%6NuqZc1#=evXKD)0{PM1ri@R$?3yPSJN1- zf4M3gILJdN^R5yzFSohf^X!{v?(E2LS(jHIcZM<6V)(g*njs{Qxh@mBO54Rlb=_J$ z$A!0Eh0aeG9HS2_eNFglP)#Pj*l3LYZ zS@M`ly11iqvfBeV^Fu_t-M5rkTomjccGBr1HtUb&(DDDBxW$PnDTg7CK5QJjtG|_O zsQ}uCA!SY&H2d%vOITFC8==5Yb~6H-V%&V?CMluLx?FT4=RAIK^=~i|qlSzwmhHFb z08@ULeB^Vc-8)AZuw^rW01~T_LId_*DE= z4G|QtzM_qXVLjPR$TTr{rRHNPyU~>y@WJ`ycxdz6e6!ok@j%+8a`~FZQw3|Z+tLlM zlC>Y4Q&n*C@@YD&z00$l4K34jsGs4#1G#XkBPi4og>^SxUi=LahSjvXd>NXQZlHqV7$0$b%GjcHDGOithAxWK@=sP zNK=0x!_}T@Wse4l1+^4b5m4C7T#9MS3}0R-SmleQsb>m!di|2T?(-nI8R9jj{%j`y zgp0Vaxd2!W7TUqj(@=Y(x9@#EHyqDoeDw_M-t6=NT=DSoM6EN`_Bd}-)jW;O0VJ`k zy$%Xpe{@u|mfNBd?1Z9#8>X{Rn2Nw~*n@T)kDH`@g7O~0+np%S>sY=^!H6XWPf2Bk zeZ^XtG_LNfk{TZ&rwV!w%+qfXUt1%G5?y?&LOsK-1yRm2#xSL4Pa+PHc>?s-V>?Jp zn{syEdY^f))VMS2fu#FUP;)fXB=ru(GfK%*b!E0Ao!cp9$}Op`3O)OvpI2y({izc( z>kA34wR%FMp*(Y3AsjxQ$)O4l_A(WWF&h+FP#7sSH0z;5Y4vx;A7>Afn2BdOOM` zA!3x5dpCJ{jL7cwbTP&l8uxpI(?oLMFe-$X>~@bydiOU>h?nwv413n09`4!@nvCX# zY#SQxnlXdrL-m`$pC(k@UP)Qh?quCE)8)prY`#;6)vj@@SO2=%Qc;aREyK@Z6mWxU zYwr}lnN(wcZGXdXHA13FJIaf(z($EfrtZe?Qm|#f@+edz1z<*x&FsW?CL|R8wdd;a z-ui(z!&qx{8K-M~ENlqKO&qi4Rys2EL%z$3F18<)Rz7mY_Z=9P;4P|dMm+co^ZFDwL@9C>ql6#N%E?Yz0rL61fY?0NOcA^Nb(LY4dH&MTG1k-OrLlWZRsCMFUUtas}; zqOvLN(7BI!pfpv$l11xBrF8ADT)JIyLCdP=5%S}q?Lvbm)Q-^*XRxDzqjk&3`l2*Q zS}U1xHW3^tcavJleNjQ1d2EZsT@ACjYgbL84O}9NFigp7FR&eu5!*|ws9n|T@X?i; z!f6w*uF{<<9fVWb^0Y(~-?haHYB+;#jA+ZBgJ)CbKKfiH&tb>XzA%ZZu}?-*?K+pN zkV?1XTt1Ix*9Q;BfX`Ujo~-WcrQsGi&%zgf_AYd#uZual$)Q%>+??kWf2ub>p|fzx zU1iM&rTa{AF^j|S3e#@n;vmh5Lfulk47C55>Lu>+|3SS(IBTms{wyWSSQ0r=bKR#* z(MlJ^Y0Kj}1_$`5{^C74-IQ|Rr|U^#YHIHOZUyVi6CV3M=-3RW#J37SVfulEh((kv z%$wQ($6mUpc^RK73~Crx%AR+pj$I<&$AdhIKu|)~DkWqurE50}ev+*<&ND zQu)9l9FaeR8YtAO0!fIwuFpcgV9qC5Y8R1*5kpSz!`@I17{=tc{S5k4pn8%P`kKy; z$P{Y%-7n(Qe$g#47aPl8{zZuH4~H*Pr>`Y=+g-l*rvr1cw*PDGi#hSOCouYY@v%VS zj68qHG6B}h8LoV?u_m(~7PDktkqVopkV}?H+-W{V8+e-@F0&~T2$pURUiR1FRGWAM zzsc$18NZFqIwh>hJrl8PE|2zlH?ouV(oX{0JRG2v?X_JS7xQLaFlX0$8?Ob_Va<+6 zHeQYJM;B4an&dRyl2GNr@?6e})j6gdRti$+>J5r^Z4}EcgI9}=*H{DmNG>HqzH<;& z#3da4ME5SMJZAOfw^#!__asJ{+onO}9yBxwu?4`!mOewf{!6pXWm=cJ$(>T2HY{dr%_>v1Wd}O<6xHyp4p12znSDyz!=EkS z0!f;R#Q*|lXMOFJoq=QB9Y;?9?cLDQTo2{GZ!VYjTjME!o6GI)e?5Om0xXyRSN!Gv ze}KPSBHca3F-Ug2|Maz7D%J%*1<$wrpUht(xv1yyqeTxs&;^6(RA<98YAYyP9HyI6 zx2*5Fx<1I(VlP^y5VS}DaJj78zSy`wL*VbWy}hoUz6x_fsy|_il8dJ)eyTJcLzqu1 zgxy@SzD7xqw}E%4$H&~d>T%*=5OKcBDt9D$z1Dkh5seiJ|NJ!N_z~}*q%83R6)i_K ze3ZC>5zEc`c1dVTUDIv}T&x1B9>iYLm!&{TG}bA4evWdly(qp$3Sov*W|O?Fz~1Np zB)Ro6UfT1;LLmKYNZ_=6EWs2Z+DqasW=+vuHg*@Pp}J~|eWc<#uT{F^DlKa_8DT=Z zUbP4iSt9;y;Q{_M+34*-T{DQz%w7OOM47N{B^vc2$hC0xi&*K$U+04`W`W23FtkdQ z-wx3)A5BBB$bQXP`Q^k^;tu7v3yh=2*1P+=_fx!B6TZEe=6zi1+gmVqcEKoi!5^&H z*v)?dxIEu84Nl)XH+OOEUq-YhP-~G35A~B_9h2NANGmrLZF_HaTRO3@d?K4aE^iSR z=PJKj!SL`;e}A30i2HoDPaYRQZKDfOtTkC+ijrL)T=qw&%q5Ll1v|`l5pq-JUA0R2 zbP0X~uxbrt`*6(tGHmPS9;1i@+4lp)cWis9HL8(H{>BlX1!+BVX0h@Ofz$~u9`9%p zqh`U^}emuJe$UaRfNmhU2Er?@=NH*MU28Mopq zzm8i#-)`O%AeZ;W_xu0-J9%FEVG+wfQmFGZ;&m-4W4#-qeKS4oT+G;qP8!+^QEoJTp1_4GV` z)wq9sVniO#2`3y`-LtD?159NzUU@?Yb{j(PeerT}90Vc*4%OCvx}rxzCo)9jQ&@@w zN_zCQUXPA|I8; z&2ayakX`Ha?8f9Cp7N^8vHARBSRXlZf0G^Q+2hBo|6=9cwDel$0+iUs=l~vub30nEPu=6`1Zlld=z`4_|F!_;^#7j#rsQ7$ s^Dlt;7r^`rVEzR#|6>5=c=!Is{pAD3_sbUu2nYZ$cY0C;N)iS7FBWKY1ONa4 literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig new file mode 100644 index 0000000000..f5ee2f4610 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore new file mode 100644 index 0000000000..9bdbec92b4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..d71222b02d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,148 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.19.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana + version: 7.3.11 +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet + version: 0.2.1 +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter + version: 0.2.1 +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer + version: 0.2.1 +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics + version: 5.16.4 +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager + version: 0.2.1 +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd + version: 0.2.1 +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy + version: 0.2.1 +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler + version: 0.2.1 +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter + version: 4.2.0 +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter + version: 4.30.3 +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager + version: 0.2.1 +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd + version: 0.2.1 +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx + version: 0.2.1 +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy + version: 0.2.1 +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler + version: 0.2.1 +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager + version: 0.2.1 +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd + version: 0.2.1 +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx + version: 0.2.1 +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy + version: 0.2.1 +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler + version: 0.2.1 +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter + version: 0.3.1 +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..45e3df9325 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.4.9 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.5 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..c81bc0093f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..1efee00e2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..725e8f31b0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..0b753af5db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..389e1386b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..b755e8cd9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..f7c201833a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..c50eeb0faf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..266f100ff9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..8978b71027 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..a8d19e9326 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..bffedd10b1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..76cc55be61 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..7f08ee0ba7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..4fd9562394 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..5ff503cf7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..7ff647b0c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..7f38498a1c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..784bb0ec7e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..04569505d6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..2827e81196 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,467 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} + +{{- define "rke2-ingress-nginx.namespace" -}} + {{- if .Values.rke2IngressNginx.namespaceOverride -}} + {{- .Values.rke2IngressNginx.namespaceOverride -}} + {{- else -}} + {{- print "kube-system" -}} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml new file mode 100644 index 0000000000..9d3e515f39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rke2IngressNginx.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + np.rke2.io/ingress: resolved + name: rke2-ingress-network-policy + namespace: {{ include "rke2-ingress-nginx.namespace" . }} +spec: + ingress: + - ports: + - port: {{ .Values.rke2IngressNginx.metricsPort }} + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/name: rke2-ingress-nginx + policyTypes: + - Ingress +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml new file mode 100644 index 0000000000..40f676556a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml @@ -0,0 +1,5433 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + networkPolicy: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.4.3 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form
    2t=zI(Z=NTeKK^m^s^L%d5$gIrco!D5}@B6N5uy0bu9pJf}XX*3J6ikcXNGYPx`_f#H?b)RtzTCOOhA6cgJ{M49 zw&07{-V1&25_jb2FrqRS*%se&9ns`EYX?qf>;V!(v0IN!Uy~`5CX#OP?Wqp!Rhf1m zZUV9z(iFG55DTNY`ZoNu#i;#t#1#DX24}&$LhoiKw_)ZYI%_zA%nmzTaW7@ntW8a1 z$?FxNAB(DkvkibvdyW`3hiWlx^MOB@9!+Wc_+Ak5vYzHqI97Rp353EW2yO%-BdUp0 z?`TbkcA8RU{;16g9u42v7U0&oE&HYZ5IvM5M9r^j7(EBHNSr};g!z30<@9qxUBzq} zax)RZxsdY0Y(H`5HL^?xv^N=kF=m9R7!3wu@H<<1h*DtmmDNjxM4iLD?B75U;nuoj z+ekt;e$CHY?`o)QZxUF>@^+I{jj+BvPDY_xH=_!uhJXNnR}8^+pJG~GWKMtF3L>1q z)5Rcl24aW>t7_|2znU)Tz1DAT2uZNca{|7qNh_5qCJs0&)Q?_MOo^Rky6XHf85MuL ze4|W9(k6Dd(qb%Qi`vUpV_pgyvJShA!pU>^SexZy-^ak%J`-}M<-vF3WOD5~LPNy1 z^WVB3&;R(G#8(u5<~=w^hn^xPWa3$hxW?JCF!CBBYAde&O>y(k94$@A#&5H29v5!T z4of1&e%|(O-UkxKdVX$Sb`WV(j>Fv!|2ZawaV#Si#Lgv_WkDWf*ssz1zcpx|{2;LPYG9ZW7sPOrC%5UsN_%7Jr4 zoQ)XQq8>lYdKuHG7d|MC*t$+3=Ts$%>`_9V8|t`}AsVVTfa6#^ZX?jGssW!lH*&X1S!EOT|4U3}Z-!F~rmZ zF^iN;IgMn}y~df;oC$L)rECln;*5#(HxGZQB{f%6uosU@*<>Z8N>h_tW!dN((pC@8 z@0Bxn^Q+hAXv+XJ0G=iaKRTC|8Rs$w7tCKR$8t(Fzd;#PE6sj$Z&FIf*hrk)js}gA z5DkQMwFm|y-*EcG>n^+g<7noaT7Zyuap96sxB}cPuJBqAjRr#uf}T zy2#Ox%s8krdJ(r1{u)UxS~opZF7|g+jo6;WsMXq&EC;SoUjvS-jr#|R2X#k*FYJsTOfeLYV8CLJF|h;6am zW%8I`zaYI<4-D{IWStD5H@s}Z>@rsTq%)*lF6lI&@f{e7y(}WR*kKDN4%Io)0-K#0&6ivw&t~ zJJLu>Koe~xD>?e({G6njpktp(tZBPQ9z=NM1TxsyTf$V)*X3g>SR^Cn(D{X(u&WqL zOs#>(VNR)NH!Ffim7cG8aVbhiF?-mFN{k5&UTTVZIG!zbx1V6J-1)rbX=4hp-vNa_ zMJIjy8mIbRekb^z?e>nuPW(Ue07Rlpz%zkh9lOmJ9H+PcQdaXnkJC^8t!;k=f^`Ig zbv*5Gv+YY2(KRwLs^VY$@H>obtyUx*2-kZA8}C3-PnYb-^}6kTIfm8*!EXd)9R$~;mXcpp zznh7iVA69nl%mJURC}Z(T-~ruOyQ(~a}I1fmC|qh?!w}>0lsQ7T5dOkSZ$|Iu0_Awf?Hpj3pCI!Dh=dTnHU+6M0si(XX zc_AIe&i&mJc#02B)iCiyKtk4u>D!B$OPPMcA^652=ynu<3ITS!k7l>87(28Azs5aO zbR4+x)yYSLQz7b1t0$j-8eXn<95+tifIg-Wf_Ur0)kR4DnwksuGW!j3( zsbFmDf4=DpJh;N$$-fAO_}IhUH;gfEI%NpQRDhBDy53#}jkM?JiBj5Tl5_`vV)uunR_ac9h8KX15>+Muu z5TRNi_EN+n{pqk3YBbQ@=kWLGw!e?0QdL_T0PgW7fSQ_4b|=S`oD$=Bt|h$bpbPU# z0+yyew6esUGsi;Rc5;5I8n)cOyx#5oUq0WBnVqt?RY^JXrYA#Ms!lTAvP`8sR{+AW zlZTn1*Qqu2mpsYw)ENCk+F674SsO`g84Fh7b_tELd>d4uUBWus*0Vj5RM6zbrk{JZ zS8G=GzsC8r$3!I!Z>8L2lV)1#CB}&!o}%vwCw967I^Xsa5Zt%Uuee1Tvc-h-ip6YI zNbRx8Ps9*j>H9DhJW$w>?Eo`eENt#Q%jl*ieMu56(cUeLe6?Sn7_4u9lBMWJoBGtI zRPhme_()WOTmQ&`h}aFq1m%*qLeks4kDEitnEo*Q@!Yh5o(F9rT+P>VAkUgtaN{Mh z5_u%f3ZkiQd*5?CAc-YOF7Fp`=wUP&OwKb~sAQdcY+vic+Fv6o5GvS6;W2D(_(1$y zFkNiEAi0(J_hV0Tq2X+;>RXh9T2t)89qIO@MK@nl|8xnWDc4PKf#W9?+VI-HCi!8T z*w6-61UG41TEvNdY~vJJ_ikllukmPrV@OoLgXnmc^!;^-9EP@=Ml=8@mO9+IUXyB} z-(}>T({NmQr3wR`{5Q?2H?WD$52I8yD-^-k?EetVsdK!U!rg492mVFHsthXlQB{x`-OC4LXw#qT)qPBU7QXt?muXv3 zUzE~h=AM@N9#Z4>v5RX&meJFi;ckUy3c(w!ZcNO*!&EQW$-RPYE|!TS#SNINbKz>) z$0tgBU;`FcBeDOH&!D ziIQl|djeSMV5oD-A@WRc+HhC*=z0JT-ks>dRko#QHQwkP^;(~Oz?|kd{vnpj1`Yd$ zK1zk-5kL8(+&ubz+w&c4%cu=qy{8=ZNL$%dj7V0#T*_uZCGh*rD*}+6yCTCB))WxW zuw;OIJ$k%b=qTz*avs9G=5HtfNU1+miREwx@#RakV_79y1#QO9Q-Aq%42>E!YZJ3f zZ8E>WGYiqyQX%Uw5j5uXbZDyoKvR&O{!ns|Y$25;>3=WSZt0y*UM9;f+6Q9}{hG=d zcHD_$s8hUWN2UGTp}_W%ICKfrSljM)baQiiKEl>d{rO({e7RhhnertTc>k=)@qH_i z%<+92(*I9=U?%!HyPQjjnHKy&=iS&q;4g@UixA4n+RJpV8}6Es`gh`toXN&E{^LV( za4@e$c6)IaP;SaXPl3vb=#qY`R)sb;i26FcQ#Dhp@53v=M`_6~Lf7pnE-P}W zA}8%{?0ZP}W)T41^bnP3dm3|Pl!nTpg~GWc=A%B#Kndp8p-?kZ93$8u-3AzH)T%D> zTCV&dzgQ4)6VICX`hzYgfI4~zjBF=njol1tx*ZXx^^sK^ff~@CY}PrO8JN-T`l^24 zVzb)f{i_a>*%pQe)^RA(RVr96#LDKIhrsG`bwIqP9S}1IztW@QOx=Nf`#g9(u)Zot zvQjMzzQ<#DqLCux<+kBGQr?_)d(Q2GF+4V zn$%WU*>*t9r>-^ zpInVoNDH+X09$5-uu+dp&AI0p2mx9F?6bPflvvvPrK@;kr_rxb5unt>^Djcg{NFN@ zf=WHrT3KNPEY?^&U3{rXtV-qrtZY_V7&s`KZ`y$KKScQ3UqS!O4?(T;)uq9=k*CUshC#r4#=T)d}_0 zu z(HO>pSj!-d#8(P$SU)FcXQ1_sr4n+GAA#wFd+8MoP7KRdGv0EwB>6RuIzb_2&L87c zs-zvGE7!s*!>E(0ls`vLQ*}w2piUz?fI+7E^Zd91T0r9keY=#6K8?`OWL;uS+o|wS zRes;Gq3=K?9qXLv2Vp5w|Bf7n#JG^M50}LF4v{FWFKLs=a)@me@P=r6pvygX7^qKt z&_#Un)zSJQ2n%yqj_)I>sV^d;ffulEF$7Qk2@{|KKk^h^I->R$EY!YyViZQIsH)C{ zL*|rJyDX&@AvyyUoid(6Oq%!NQD|D0+nJGI3R9XGZ+bz->{{3e8wGQI@ zi-Z>8YU*>e@cY}e1PqsGxERiu_P94Wl*!8TGL$Zz|jpt&&EPw zD^Tk$4_9-QLe4N<7}rWpnrObczEP^53^f1QZ@9*YWVE_t7Gxttt4(z2lcfrd7_ez0+I8lPZbdh;7a2&&avbb{E*)4j?x$$ytXQYXD4E55o2~nO!3K76xTreB zspD7wo}L9i(R>JnG&*WM-#2sJ29D<|C;L?mjO*KI>H^`fvHz1|I%qc+jXw{0*VlXs zor(FeO>Qhn&s8Qd8lR<`5zYHCcqD9BX%C$chFluKntb^BA zyUlPM%DeV~q^qUfrNP_x?enE6;@Q#zEo1VM1bx;~{}WBm_pBWT-KCO(>HDSoGaTYs z?$J@^@loo*SLy*Q32HA6dM|?DD}n$P26Yq!edGu6M^R=^RKW|WweDH3Z2f?jf6we_PD`k(lCn#>`Q@q z;}ujOLYEBAs=Z0Yt~c|p@W0!Ee>0+6;%?XUXiGd4{5_#|q#ZA?j#0bmg_`wqP3}`^=jt` zd3k{juR(jj$72|Rh3F*`KR}4AaY#CMD)q(Cfm5HkgWmA3il4T_8aYz=ZA*nxwNOcH zs^^V9*AyK2?O&H#!3iC>z6JJd*x@eo>)QN>^{GwyvkHWv`h=kiTEpfg^bAiuwy$q4 z4dcyGE7@Pit%?&0;2CCsoQ`zHogPmtL-bQkgVkn?P{u~@_JYlcYr`I3T10udF>-UW zmC@SNuU%wh%66~zh8Ujw`*V2?jR8~FeQ5_sc!#Z$04tMX(^l2xk5f0N1JH33@oRui zSY;oF0{QtPVi^sN1~I)3;ec!zb+lP;qq`zaS*SYAX)jtv#s9jQOyEK#xu4AcGa0=G`-P~o{wm}H%sfV=#fCq~P zyzCWv|9jxM<1aOq5c$J7o1rJE6xM}o@D&e$=1`A25LFY{O3xX4Be4Y>$ zMcWZU=DkU}HmQ5l%gzXPRRHjNx02F1om;MqR>1B2J)sNaO{Zt`0<<^dU;G^S=Yy*B zyY}J^ffc)-Dvu8QyX_Kq`vPeD6SglBtT()04X2^hljM9 zX=Q3(Q_Q^kM8#bJNwgw^bqbuRiS&fVQQ}7ojqK+v?@MTuc%02W9KTx)F%{&22mvnn?W!@F91aj6iox(3iH4Uwxz&-wNP8U zu~bc{0dlCKQ2JG4WB&pYM=8(!cSf#5(l$-aZ`TZ)``?vMs|ikuNspX`V-3;c^qN#O zG!X1#XeKJO|KQtmwQZYS21BMS;Zj8Py|JsnwTsncmgs((-~`^5z&@9{mv@+k=_*M` zsZ)hh`pkYVJe&HJpKjj*O zcJv(~K>xwC+%>eR4#PGvCmO#=b>5e-+2HX-l6J*WYNJ^?K#on=AO3b$*<59jq=a$j z+j2_L!Q}Gdp|=Zi(kMUq|pUR0KiaI~nZ5bR`K zPA$sAX(tw849)vqR=SWVaF%so49gV-eNytS+7BUjGs*`;@ zPZmo_)>7i^ea9o%*Q8daDeB~H#QiP+cDWV0J!*gyi_dog2B|@iK+o=_i`pidC4-+R ziXo|X=n|rnf7_g7j*_E-0f9Ho<3uj4ze5R43xl>)pP26)N0y5;rS4?>Qc1}L+OPyD6UMM%Bb$Xi@Trp5aG z3nh3K()UYCcs-~ZR}$7BE-d8XN?^FpXmy>03GOJ6!V|FB_u)pd^$*!wff)f zW+V)esuJ`lIJ3=utcc5WB5HRkM%%u&*rqkt@`PB8>VqL~DoLorE}d4F5IpvwT-~y% z`8rU>ejmpA68-CYv$3oxYjkPY#IIL&S!{K`9Ibuk#l}L}|BO`2*mMf!<6Bel=8LJ6 zOMRJu-a0yoLsI0I`toK|JryboCP&Wixa4+KLkQ8(Ce{F{@xBFkWO?nc0!dI4jNiN? z&?}_6|26AK%A)1oNctkJUHu`RwF&M}asi%lmz(KuF;IbAtK%{dDdm$M^13 z!8}LM^YN&}SHv8sUEq$PY zywE-^{YRp@AbhGH-#}lP95bhC|GL<1_+y2188Up+%q=?NUNN6b8@C8y{yR49^3CRA z=D0&)sw3)VS+u{C7z?5Vf5#kC{a(Y1yyn@5qqM|Te*^WJ1;+~-+E!4I%pU*zY* zw_rd+OvZto{A}-z`FU@-WE1@BgW#vlxur6@CdoJEJ(V11K@24qlgmZCu)m@c)@2SN zrM-TwD)7CIY7tf@JGS@wHfrJO4MJSCEp}?xa;Hc{MY75b3K4i zzgG9)F$+zv{(JLZ`PH|jBjg-iE}yOI)Y(FuCc1{Ji4f&@b1(2dGRjIj*D{G4{Cstl zrrCrB_paPh!+Y+iEPS^P%H2Dj!mvPoPVp5IcWJrxaE7H%L%E=U2kl?) z09b%JE`7L-7ObBRi%o%of(?I4ckVlVsR4%v)9)rxHf_bDSDpX;aziGIoB!Jv$u2gL zt5=c2MrVkpRHIP(M^eIFf}YCDX2@LeXdL2sUn?5C74PM5>LLG|4Kbrqe}cK1bKP>j zhN;?~u5u-0ZrgkHMSI!Z6~c3XCSzbJujzc5$)ljSyv1+asCiyH;mQp>?fO(|EwXU* zNms7qG(L~oL%g0^)2$@u>H*|Yej;ppwUxYdg7GS(B@vZc@G(oH0WxxFoUkQ5j#5vB zY4Zl?EWOM5Vw-5a3^@DSBv8S1w&nQqh(?`Sgf0q~0bI5B`ag{PQ*bZ;*98p5C$??d z&WUZ?PEPWTZQHhO+qP}n#?=4!yfsr(H5YT!pS88RFLte}-o2~Ki^3e|lS@{lPAqw^ znd?av{pQhVgN6$j*GnE}p3J&qLz9;r&tzmomJW_t(+$dPw?#@KsfNm{0rG?w2_;m9 z7CHODPW{sXY1lW3u(!u>t7nPpU(}Az^e3b86|#fRXQb~W8ujV*g`<4QMtw$ z&mtT~Az_Fgku42gBN#9VH(N0WTh*A>=fae(-`cK(ms9|x2GKFx_7mnW>S#e8^2Ajx{Ex+fv|j8LMj~a7JNO#l5mOA z^p%A4jVV2DYtRqhx2oSx101~rXZ={8G3nuF4x4F6{@leNc2|;PC_=)U$?R=o7yhfd z$(h@?CHG5v6XQrYJh~$5zZ8x$q~N{C;xt~X&3Fxr>jx^>yJM)`dvg)6mCUxnk(pq^ zUTtRjN3t`;bn7$4MJySL%iBkY(_sCsWn&m1r6TOMNXnD1fR2(C6S}dCgw-EbJwfJz zcI?B6CnQ2xbYNvOT*Z_-)fhBdFVEJFG)j?=q{eC*8%I-T)y|-p1Sp@0-OJ;S(9p_9 z2NK}|k}hpvkG0wI9r4%|{BBKu?|aXBmsbP#ZxtVc(wbFb>8IA<46Mz2^h)};)toSeljlnjIcg8sTt4GB__JFJ|WVsv-HH?P@5Ug2fo3c z+Xldk@AMD>Buu#kBVC)1$FQ!%e#qmX3`XvSQEwOYWmU?0Y6Qf5J57LxCAV=_5g3ea+i z$x;h6VRDJJUi&G29Q&rSuNDgNn;{Wo-9OUExpmHc86f z@PL`5xl4op2J6k9EYt*QQXgnz_#ZjSC4z?b;W};h%rzKwM`yzzT)%bt`R86Kd+~=h zUO9v@GF{qjuFjfj*{OE%84YDYBQwJx4SO`ks;brs4CjEl zB;I@jqKH0JmW(s=7Oz}q#Za`7Yn=*dZyWk{IV)m=0OArNT&dJsy!#cuhv=P0%hkc? zfH135oSFz6s0KxzSNvwcG-d#lnaIjGn=uc}OOC0-G^QHuYLx#n*L>eVyIE6(*H*7p zO}3}b^%gKngErq*K& zO+ODEkdsxy!X4Q!5v zuYcospdmMgi2p2U&Jyt@n;s$)oGKAMP=jhMXAgRPwj!@Ii*e{Adl7d%Ry|w9bXd8{ zRSvEbXh9|VDnYC^8KlszpXjM~oPU+AWs<)bmc{=rWsXtIbCjUb%!t^gd3T3QmAF7| zsZC}qQQ+2bOxNPZ<~G{1I^L(YG`9BYO?!tQa@&5EGaV=3W-tNxviRBA!FA1QHcyT6 zg)ZzmHmneX=UqBf2U8#DtYWex=GXoO5lt9e$QkTj1FJ-vi>Tr^g-8F_A^ROn> z!=r^;<(z4&kn~vANlbObj9xp^D5a8Alp7k# zBNlhG`W)a;KXU1BeXYl$pDu@Y&|2=ch6CB1afNz#d|)Nu-UV;c7&UNZ3iCt%+670# z8Cn%l+0fOzrYe!eXXcHPQsx!aM@5_>3Xee3bTRtj28-XaSBHB^0!Tze$18?nqq+d< z^Iu|G{L`S-XRc5;C+DK%K2<^yoWgo+y^7*v5t43s6f3Qf23aL=%{#FR&I6KZYd0pyF(Qw+3oN=^z zO>GG0Lhx)ix!DmJo-NWAcZP}J*y8eEkTiLsdcL$z8vG+`qpbB3*WuL-HZ_(QGvR{P zn0oOS*LzB=rp-Bi{2FWK?K~@Jr9cI5Onikp*Yf6W-1xT8gCRBx=Lj=VvHztC4caPO zgeC_PqOMwPMtd}$>~8@=S+-NQrh=lE>Id3OxPQfAFzo$*C~`WDsf$*Kqqf`N2eCY@ zaw8!s%|*x--kL_R3@1kTwW2jBDU`3YyKR=lk*2akHAvl~{)YrCkhk zBD7h{lJ@gtUxB}>TM>6(&XD~>!&`VX%?k;3w{Ie_)lYdEux^fhcHFil=lt-r%A;2W zZ=Nny`RI(_7}*goXB6JxH{5Azlg(FRicqK}_p8 zI(15x#;oT(j4og{f4)Bw`y3B~sOI?Wa+i~@wS&kyz|%AGNq~I3v~AV1Vht=iLoWx3 zGCH+k60X*+Im7rUl-QX7IOT%k&|Mxpv5d-3ZB1b^NPFm%a#ZG_6I1x}oS|x-#5XEZ z9O?C9IJz8Ujyfg)9IF|PMG1;6jmC~;?-QLV87Q6M=_KQ59G04t>&jwDZBUAmGu}kr zn3EO*GE-YiZ}t3iGf<~3-eP`e%Za-MCBK;mvYK@vy5lGwuYo(*sE+A(y@m{`(kiF6 zK5B)fkr@^}3!Jf)&~Pd^?rg)H^lWKz^JN#ykn_9mZyUG2XUdm3SyaA9tIJ_hd}(d z%C*s;Sq*Nvh?XvLf}=_-ICMqX_M$`GInSEF&-}0rx+q+A#U!N#Wt6&1^Xgb1X^NEF z{vIu~A;5vL8X5{7`jz)X;MY;7qrg68nVg-wP|SSkEDAby9t+&qNi2G%Y#xh^yKoCL zTd|PE%1MlE^ek#)_wdzGOx0?}fULa+uwh}lc!NZfF=XxN9K7*0S<5tPIVM}poyR2Z zUu8WQy>_UiUvs_<=m2cWF>MDr=v1s}&wYD5{xQZ^YH68t1$;=-kZzQitesS|D47C^ z&c?8mz5OSa>+qm(fYJ&Qk)_bcvmROD$FWrhu={&W(IwL{6%EC4_r1GG3+nZ#b#w88 z$dJJUf6TMiyTT>%G7@lTYP;3Nb_1IO7u2)WztA~}fo!pD-HB#&q@~p^p-pRas@8GW zs&&s%b^GWlWfgu^bE#P4BT4Q+8Tr&WGhx>nEzIhL$vRg(YfV-oc7FSy*G2!d#x7q? zVOFlM&sUkLbFG1xUh^VW2DsLnUeCpD+gfZZ^x^lRXq!ZwXeLLPmZW0A^lzQq5AS&K z3ES|NK3^JVeo_;YD%fYfhBQtk)ja-n5(7^ZSzE`MjyA+nmY;-pP7c*8AepvkjDFVT zxi}@`+uYhryImQQEGUr#j1R;f)$5$ zpM;_j5e*WhkObeAp<2N}XB+EFo&?>s5#k%48Q--!kF2^EnZrIE!r|;ghc`i77?753 zcd)I&uFqJmoE1cuL8?c(vy^XpRN_X7p7B>D3w*^w`O@x*kBwqYujwnd!7DV?WI51B@it4R^$!)Tvbl!)Yp$j)yfWl7d9=8f#ZVy8(|zS`&O^M14R#snY>H))2twg11S`T1WD#m3SHRx|z2Dfs)p zyYm4qjIY93UWTwd4^$2ZFTxMsNoLAPjbw*VW=@>yyK_d0xp{j#x@!3Txhx}nM#uZ4 zZY@yhl#yv6HDAg7TNn^bK(t&sy7g105+dd9?GQ0j%410*T%ze`qg2AI7oRX%mb^k+ z9Sfk;*u&!|(%`d+lDN)X z6mpS}WJI#3#fG#d<4yl@O1c*l$s!LftE7#mcXa@(8GNjY-_X8SNhLHJX>fBEMOfty zE{m7MgM6nhnxND@37X>cCRCWJi}(~vBqKZT2+u|A_=8E8JaK%uxbiIJPArcygPcI) zUUB=Ak+51?+ul}a!!*xpYeZSZ8yqO`jroJJmYSI=ug@=bYH7vejFvDJImOCO)F1jzD5YcUCzw9W;Ys&U?;s>;=6o4s^6~;}kgB88z*;u>xA#2M z={VYlEEu{wUGuqn*$ZLvPlGtdig)!r6 zOm~|@8iYP$7|9?49&M+g?m7zYgT81dlOhrlymy6j35k%(q8v=Bn^^&)e96&6kH0BQ zB#pv%n8Ha{QKV}b0Usj&>V7e~uui%9g@`8<5|C)gBf1ooolLpOF%lX?wQDke+W*jx z^|cckoHjQ)8CC>Ty=`zZGG<^PGE+*3)sgZI=mCXl6B|!Ku&d;i(8(xkGJg11&tVaK zhAC5#jAo{36#x+SQqIp^xSixZ_d2p)axdjh>y8>qx#f)?dueLnEpRwx!Q&T#NHc-V z_Y$$O@4EGL_a7>}+-IWiai`f!RO<81Bra{j&?+@|p&(L5@q+7bC0xe}oFb~9FsN zzx_nRnvm`5+0a;=En0PoAn!=rN8#r8dJJ7z40GQQ<9<3*EsCh&af~(>vQP(EwAdJ) z%*rB_W(9^$8Dpc+jl`~8b8ZWQAop@6KhVyBB)RChUZl0bgBD@5qg&Id{%y=4ObJIFc zetO!ycFNPx4sYKGL~*I5u0aVu+#9q2;#W@(acaTGF67GVbjyj|Hu4xTItTHzRPDvumBs!a04bRAS=&0 z5#c_IrJ{{fP@T8=Hv~13rr*5i_pXj``a$?*grX<|Pr)F(&5-VIjI4g}%@^7CoS`0b z-eAP+S_@fFHMs^4hVbpbPY>j@lq$H^gtNu-Q+=6PnF4C}?c=1`dL@y%(X+GOnH~IZC-(PP{hzPJr_{sR zyQo{tBa$ia?w_^T2}o}&u97X*558R#h>PGzW{Hu4?KHA!BP>jQnYAZI>U)YcCXfE z=Y$R57UBkN-z!yyfYlh7Q(hP;!3h3y&eR>(0{8V_e>#1m+JC4kRo=4P(a>|$lGS=0F-t2fXy~UY! zp;y#f-qNxbIZ%XyF{a~6kH5vhbaQCR6sDCtGJ<9^vO}UT>fGp(s;kBQrf6K@$^b`pt zr2PkKvuDQ{kb?V+*7~JdWeO^BHi2&WCIb42P_u&j?$;L-Rg84|e{C|SJDIpDaV`}T zx}+V}?m$F>be6GUHNB{03d_?8-{1|w)3*?_OOr|TC@JUDorVcs^!C-lnYD|+tavq? zYV5&I3Qu3LGc!mE&eX79(1W^*r;Lp>2z`%#Nbbh!G-vkM67@(iY_j&;KMxjy#4mTMVia`m6ox#nI2 zyT+LozlsukCeHB!5Iai(8qv-sQyu3{r&m4t)zCP><9W3cpNeW+DT*pIGcYC+tbFU0 zWIxjKRnJlRj)V2CDRTRu!6YMx5=293ayeJ5DV-Q}s`Zi5_qM9LTWlr%r488O3lB0m z-AP`{|M^)z3Am(6kF-!rRTKrAnG?jDK-E37OKftn@BX_#!`wg)GNHC^%p({;TPMLn z*E8r`oAk6N7~fEZerTl?*}`^xvY-JPKJXxygH6vVSwE1tq07gHd`h}siDq^VwzD8(7N2ups!)beoqaD<~8Ewec(y z;bl7m@0KMCQ4IWjE_T*|{ZAc&PhY>6pAoUQWF?4&;4*?hP$ZAuWo1sH$2xorB6 z;4-Xz{Mn-28|S9D3W(AAkveG=Ob-U7h~ceasDb1(_BCmc-w?zYV=ax~Aqu!$W(vo5|gzhk4H=Ta5I?)8H<8B0Q+rVoj19 z(r0h5P$#uHuiA2pQxYDMsuvxZxsjnWw~7nCKqJK`6NKiGMoEK;XBixltLQa^hK+rN z)!x(r>Lqc9hK=eM41s1CL(|7wmho_NY!cs}v+s0PM=U}Px4BcERT`W9%nVVhE@D`1 zC-W@IY54<5pN-T;_P_6Re(J@QP zZNZTaM}<(sse0;+@F&nDo>211eCgYaMdD@h?{v3iQlQETp0ku9Ec)N+`vyCKiL;1} z9aMG?p#qY}Ny`KbrqwO}ZekjGc8VME1cq^`N)VLtwmujKB_Bd``}Z#3?X)BIFZ*cI z!%zw6{No}U<>kOl9$RYpA%bjGg6To}X)G~zS^xnm*r9f8m6g6Ey_lkjb+v*McYo{X z96E@Ttf`GQ#uPgP2|X|~TZ|!b5>*4_MdY+A#8c%N;zk*r{1{Ae@&^|N4J9w1A8!H& zS8CqJ2G8mNT%HwOIMX-x}i%{V`q1jh| z^q=Uwx3T8vl)z6Pa&n}Cb z>*OpZ<|@Wer}^e-inr8U9E3EaaulvW=xsKU%eNj3U!ZrNL8rkbC8RWj@)ivQE4o>q zXX)2A`bH?}qCPivk=wt_KwpI0?bxujN=2t5VYV7oc{DkcqhZoa{r0lLV8F&MA_N!$fvBJRDX1v32udA>7r@646|fB zVT)#zIzyQ!VQ7XpxYUu;nags}rUC>W27}rEelrE~^ z#STrSH(Vpn@_2^Unygpp3Mn|YiGnBlQwG6=OLwZIYb`n{Zj`y^a;zs9=bB`txk6!*f zN4gL4P>dRy2K!lhF#=Q>mPcJ?<*q4fopOI9R}eszv5xBP6N=n3!P2My*X~#BD&XT- zfPySX<|*~~d~`W?4}-r4m*yjmgA}U%pk&}ZgtS==H!jka8;TnKmB>U3&qx%$$oNU_ zA>18-5oTuTD@_qXeZIs(M04B5cpSnC6RZUpN2=VfT(bRS_NQacpD}mv(j?^=3U8lze3YH{gJ-eL`7eth(zL^ znl&pO+!8hHsT=Nbq(JaTa>>+}$-0p{dmi6TXEn*9Y%f-p@i2;WF6^+ZBy$Uo9 zhBY2cUE;6Nre1cHsk?=ULRM*w%AX}9M(eOaM%JvHf;N-fk2^a@oz|CZbDUdJ5yMig z$N9H(ga=yqeNDs)6tuW?&6GKGiH75XJ&%u04BJ`e2TF$<^_8tK_$6%o5;jhWN|IN( zwVl7Jb1e$XVk6^P+CS465JfBxTx$O7#PFT?=U89QkqrEwMph-~goYvXv-(-T9^Zo~ z5-|~6%bufZf^Lqe&K6n%`^_GzCN!=Z#~kbsa+5sMGBnXEo+GCk1#18*EJ7g1 zmz$XtyB3`?%BusbSiZzgD&~xJVnk_B8Sz{gVAGJeik3TTrBBq71Df1Ake%$eT-R+K zP3Q(yU(hhog7_#^P)BFAx#A?+xc3pG?42FnL|IY~);U{nD$W%-j>=AtM-zYNDDDrA zWzit;wn(Z>XUu2#pAf8d7ccGZ>wqh4VX5aYUY?h4M30EUL(*cv;26|@zehoS9zE6dE?Id)2b*8{1 zN^{u*=i0x739<;9q|)ZE#v&8ScPyyw^Q-76AfSqzsOxysEBp#cu=+fy0$|TfiEFmKu$IHXg5{T_6CSmPublJS6U(Q z&|uB75@79Qg7&hQ>r5cszxwDYytD2&dJP&#yTxI5AOaGPzqc9-V=J%=W38SPqxmc5 z#o0c}#0XTaqyC4Rn>+qyM`{XfGw>iJ>=zp?!~LJr)SU^38)1P`Zd@-CdP^Wb)!L>99*b9!wBwTJJGS zKoE*%TIZO)-^YW?yxc5#!(aJjHcFO7v97O#&C!BYYuqzrP5!!#{n#e$RS2A4e-K}rPnVa@f;0`}Bs z$u-!}@=3BQx%?D0%MW*BC6T2W#+wu3^LcS0869GDB0*_tozo9*@EB({@<~|6WTdxP zQw&~K5^M(C$j%m@ZI3GQK)+v3j|mPmgpZ6}1i2v!yAcPvY1F0?WW7*g-G-D8eTr4% zQP&^y0~$7wNG5rsr}?`aPg;bEgswstxz4&@sT^%`@@s~bn_W|PC$UbR;duS$21dxgW^zq^>-z-3Wrl&GgCe%XH#RusVWV6EU1O&o{BKOTB7 zKC*L++3i$^z$_IL{wlEY?ZZT@9(Bu!1;2vrkzU>tqpD-x=hUxms0RKpL zqmk(-&p#Y`M{%>A$~lM2+!xx$@Vg(V3K8>wHSz)T=MCEUJBW#Y;QQlGVJo-TP-Mda zT;m>G!w%dFXu)3zN-GY*N3ClJ$ec4D^iEIB%Y!c;VQ-(2TnwET1>apyie5nspDaEy ztR0=uY`tnF2kjfW!c_Lcz|^%_wCwJ=j_0iTd1e*+GM zxxu$TzR_v5u|C*k^lwbXj^x&X}m zCR*oVMjrg5x-SlNJ7U~K-@o@K+fX3QOPO}ya%0PKI%B%g)N7?iVq^s@LLl|TA8JWd z3UQFE3=KBMPqShV8RO%A;$IjAQzTYF#LH&t;q>i$A;NA%8~D4;J!FIx5(B6M(K6(c z7#`BBq6q8JR!lh0;Xp@R7*N@!I88&EnjH9ropJnj`~sq@B3O^bhWl9NFdB^P1SED2 z2^MXVVvIBocf4%98+Sr&?sC$gmdgw&W)$(Zx?PYOSg=KAwG6u7a=va7w)8be6;;c7 zv*7xROB(vyNJ2Ei3<_n<`U@Z<>p}705c31bl7WdQI3su?N@YC+34!iAV}!cOi;4=l zb;1QCBS@hNB>{@fP+0TF4GNUUwd?iGKMLWD8A!p_{$eR(5+ay6rbGj9_23-Yp}{9$ zl|qP3eG#xJT2nip^L`Tz6o7uxduX8gd_a5wye^VnS@*d_38*U?VgCN-eIQ0P(4siH zNvZvhT`eVed&aEG#%4e-x`UQWaZp%aDMO8~x69A-aHCIBXWw{(9zhBbEGxLhFmf%_ zK$*c)FU%zZ54EAkmi#=MOJD*9Yd+r68~9FLJ%kQ&nNBa6;k{4#fX)8M*dyVj;N9V3 zD#j{eQaa3-J3AB9o$DJ2*vkt@^YvZHS(fdT^a-_MI5RaGaET3F=nR>wJ$vx)bLXmk z1~i>f3Re0u7x#*BkOKFl6;6LxfJC%j+pN2lD^MEJnhe}&ij;vUI;$WJF*?|>2AOoz z08WxUQN$KW#d)$}aOoKi{oKU1qjz&t8({kyzM9_cJ6B~QU~Ey`-1qYdCKmULP}X1~ zkJ&zVwP3h@Q{A!WgV;SRxX31c8PKrXfV>G=lCk1vhJaS=Y8Bbd-nFt0uN$&uh>k+n zOYaQ72}55?3@0~-^}*v2_yaFBrT?uY?Pc+|s>eKo8(}S%;F)gKE`O4yu1tXhk>o)P zDL)ve)6Ua!HD93V6qJ<;KlQN>n0)|*IF=D;0k;ml_P4*-!;nF|@x}YOI=O@tI4rue zG#I>`dMK6h7;)dgXgY;;FJfE3A^1BdICM=#N&m#>xk$MX5FS z8?B9o>pz8A{H0g7-o9Z^MRDCa)f`E*SMDnzJnFITDW<5?LW7=VW)@9F43FZ383Ig0zNuH& zZ{$>S=n)R^ZH)gIRLUN)7-IKxk%Tdlv45!4xOr3~l-xV@sp8x|Mm@Uz;}NTH`x0xD zs(On_(KP%4rAS8FUpVzFOhCqDjRN! zHc67tJY(*=;Hrf49EsuLXX3F<=Etu>PWaSX_*xM#ONz2XKT#u}l2q9X+t`EB+HFfkgkdLSnvZBF#^t+k2*C#ec*i7$VNNNy@#91>)4-Y*Y(rO;4HSzwce$y!cC z(_Z#m?gj3bOB9&|n)o~gT_3DIMw zgLmcMg2C}?Lq9u*>e>M-60LrG5oP`7p}yeYkd~?Gt#DE_;~DN+K~}I zEyN6r^GvE-<`2jV{sM~(#AN#ARhHJHtum~3_E7S_{mY=kEwB0*uh*u8qLYGXY<@7b zmAjf&VJlq-@XchY1Mu>17M9F3E6|ATl&w^ioGk=#<+A2N-rBaWWsnTFzqGuLIwH_U z2xn9Q`>l@zoA{xC(bZRp2AE_*BX~x~36_f|6d|hE=Gd2vW$ID*Bwpxq=v&Qs|3SRp zIx2Ygiqp;pG6!Qmc!C2F6hbzQ0V|^}8>xq2#uU}?s|r0>C!hPkQFyCZ;cqV=#|Og5vzG#-LZ|#X3!wJqL4YeQYT2iTa%wyVQS9 zCCxB*LxN{-HLG^XY@8_GYNaCbEkN%_f*{uCTDrveW%6)>ZHp$O-v5kF$i*K__I2=i zzI+y__2u*Z_<8{M=tl!ovrAi&P+fQOr%S{xFfkrGK)cX{&6`vC(skFs3QIouBg&Hn zf}LV&kwc(%RcpL{ME0dEJh7tja-g{MBd{qKxhBhNM}z9tpk{q{z1Ii#!5#>Ht972Z zZ}3Mg_5t3qk44^N2TubdRSOpl-8W7xr4~khf^@`qNQZd=d)Dw0PR|b?55G7MrY7+Y zz7XSYGN94vfiKU=)w+x~SKo<6(lf{eX`dAa)6oCWV4Du8(hDlf2&+yqoq@Fxi}cMS z0w$CaIY5Y56ddvt!^`BlPB;_!RLDn|4cE78RLEmataskCFlhOyTfIPwg8-&v;@@m# z3#4uVQE1e4uJqXjf|bptL ztOZP`;w14cTXlrwhJJj&n=w5qKi8E~?nzjX9+e-pn>?F+bIvH>JgK|TdTv~EDSpq_ z$I8(ZS|H%TVN~`06I|M4%)&jYB$U3jb`qCzDgcKwt{uWGNqmw-Ee<8*x%!7)RAGZjv0u;_p0bFSIL%#vr(wvr4FVNh**+2T=VzjlWR9X*7t=V^;1uC1 z#vQfP7JUDBhQ!j6CJS*OfF7sKVS!`@#Jit-DhlkZsgL%!vRcwgFpV|BL>?!I65?45 zcyB-IFP3KF-}PCSV*BrqoM0X+Ab)0FlJw|s*8X`zlzze_JY>F+MnlcFS~oEn{s`?P zhP-TsBdxa#Bg;o{3tp)ru2Bd;|Sz zT#J%_Lw$S3dD*gmHI@?Q>X=2AO$)yDepN`u%`-(lra3)H zb9wu*mhcb=$gK!0K98xnX5D7?T$zwA#w4aNWiVz8#@wW=YLro4J&hB(o*V_k=xY@m zSx%c}aOTijhXrl-{RkysasFbqZ8=t@P)Nj0nb2&(voF~)?o&L9|W8kH_ z0Av!}PUNpr5NQ7I7L=^$83>a>Z)XW~muWMtj0uJJu~YFKw6Opg5EcFq)39{%zWhqo zw$N}Jj6_{}_J>kmp9DE}eik-4Gn5=jd}rVYu>^t$TgJ!KM8!NW^Np@SFz;W&HOwO) zDC^Q=js;rRyAGm_EyJ!Q->_dKd4#Mcb9-dPL!tZf_|aW zXtJq!ifuZ7AH;?GmXCLVyg6h?u6B6J;}kOPhw|m&lnZWkloUmVOSdZStzO zkN1ujUK3vCsnFgnWGM}`Ddm+=?5au2(HA_PyY`_BsqUC^Kjq;jMzRDL^LiCqx?MK>Vhx)Azw1tRt`SXDvW*(DI>c8i5%^QN|~ z45wgQ2qyA9O8LJIQqQB z_%@XNvd$TQQVE2uJ_b;!vJ6eX@k3xqShu?&AT#IaNUqA1SD}~43qW}t>#b>( z!SelPAS5u2E3b@mRSh6xCn?6T2+-85(FB+~)B}I91F2nZY3wDNu_Le3<-~&r{~gtv zBOB$cDu&3yw2Y%}Du^sbVn+&zkRIa_V2Sy31^&l~amI#DGktI00Cr1bMo9f6r6*`wzIJBU!=h zymI^RjY8L@xkEod-ke=|HPe zPzARBj0@xTw<7&EM*zO&Gq9DSUrIm`5i*1eq1W?KHj?T+xwT4jdtJpCgmm;6|IiKO z%iKy%GkoP9Z+l9$VZ@22WilS3(pM2|TaQ8vag(bOh9i+&CnkZU2~IQ>D1So&Ihux0 zk1u%PP=X3jH!B3!G_>OPwR>wSxvqK|EK z(Va)3bnc#59uXN?S4tD2PBKYyyLvPWDM`Wi>aFsbx{4^)WtJ91o=q0nc>rAon<`b5 zDqfB`f*P1W65w=ey|sV(22g}3rmv@oU*?vp%2$CrO-!!%HZyloG7LzP1au~J+sMOa zO>C%MfkTB2jAOjPItJG&4ZOi2^?^Re;wjV8mE)%7~K zELdz9ZDvAt3R3%w1w;w-xe%wga2hP{In);v-`j)3Is(3wN;7w5#x~)8xT)}Uab`Gx zM}f6Rt*a6(DCMiU{B_0Oa38im>r~XNYGl`9Dg=*)bbV&=yi2|Z&pA(fX>??voyq|x z@?UjuM#ys_ap|m2V53b)Vkgc|6vL6U3lSK`bi28R^0R6GjaTtQrKPEtVRs9?O1rhN z>`c8Wu^#u94y(H`jdK!F!Kt-M-iPaFc%7(0cXe)#_Ya4oEC5eHu)iY4nVmUeEVVqW zAV@|9)(NzE`btB9nA-6EUMHfbd6yc15|>VCXX{IOGWuIZQ8Hm#UM57Oyc`b%g}Z49 zPD!aHa`Adfe}$Ju?TAWqaqYx0njtYq^1cQ0#0J z`jpEQ28Ds)&YDHH*5?l+Yme~F!%l}5b|1gmPJPV$)aa zJZ^|1^ugVP4dx=C-r6U%t*qTDl z-hZ86ksR)MdFnKp^rKp-9LQ023;{+mi2~2J!>cCYt6&TL79paHsZszZ+m#TrV4ndI1v2q(JW{<1x3M*mrJZiS2)W;^yrq`rfxQS^Fb489YJdOC!Fhz}kf>{Q_1PRD#GYA>XtC^0 zV-~Q-Lzi#VQoj$QLgx-#Kz`KoJ(Y)ZF!4?@JP{(gw9xY4uql0O{9L&ROGCU|zTKn0 z|5vV6x!8m44}(XD!a15IK-6bxVT8tb?zZhaTSIlQSgR-OTEKuXnJH|!G5b|q9vCb9 zzcHG5UA;g~uLt1$U~{zCad+aCUJi*xm$a6#UM-_P=lyd`nY!oBz)r;u1EzH?H4W_m=>9#^(?; z8Jr4}8ug4yph^VggbLNGh#*^%g|t>%mOUa^TJdOM@~Tm)Xq(0Eb96yS+qTnREubx= zQt|`jGy*H`; z*MpS`Hj8rOewHBkJ}^AQ@&W-LeUkG*s)ZsLXLluZt0MY^Ae`Xbjzu8m7zg9r0I)V(`?wmggpnKMqA_sDZGWbm4{w@@v<{=)gmqvaD;@~8pF0voc0Ry zwyr++$@~L4>$h9h9#2r~`duYp-qR}2mL~8PLYPRY-XaMhFo<4aiAY*BWjmwSRVuVf~}Y)!>dS1^v`UenZSOy zyIr1$jC@(_nQnnwdgD-6J8)|*hyeN+H;9ziYpq80a~1lUAky97yEV`)*>5M0g&g{h zm#O{xD18xKb>`@Ju%2mmzA8t+8&rs1H3O6jL~7m5Nv(=XYn_ZuYUZpU4u)~?5*izy zNDHyaSCKVwLm$lwefADts_3~%kQZKV3>#SzbW+xm7owMaN};!*KHPvl<6Kqq@C&?} zoat|r1js+bnc_AHe7n7oM);UZi7LCHJ6ezc1B7U~oe{ner8yf~ote&ZkE4V(_y&q< zTxd8BNGB;q8)tnUk>Svp!HksiB)VrOZhi^fc`!3JE1*7SiH#LhY@~pY9^{#wB-=kfm7Cf zpGC7dk_g?H$skj&QV0jmJXTMim(xtyvA!X;#u&`Kl$ar`(E@ zapZ_47oF!OMs9!6w1sW_Lb0C49Xp@ZL;dxzj?5ku4$T_$Iacr7hHw(P=V$(uC$`^T zjb2-0tv<)hH$GQ;3J(Iy{FLy8t1aAHa{uB8Lr##SJt1VhdaxWlPpESD1s$;E8ecI- zID(&}oWJUQ$3s-b384A+D^-T9gM40;9zNU3HNgr+#avRh8xJUFJgHGIJ%kkJy1@oU zVHlz=;5?all^Un4+scUh1sE@}OYXbWulKT{om=s)w)lM{t$9F%R9Gua#d<5{ynq|_ z18=0JkZr-b{+*^f1HNWv4Kx$==Kb-eDC@CnJVZLZ8m5^R2PLyimLx=~#`SX(w=}R6 z2O`O$XZLPFe9M&_=1N9ViCG+eeam6i3Up(0Qz@)}?F<{j7DlT(iE$z?gcnINu6atd z!HvXbmN4n71qN)cUi9_amlS5dLE6xIw}cpia5K8Jh{;|mSH2CsVVp!)1a@2&6D> z!(^P(5nghTBIz28mfvOt+_}d!&Rpy|@S9P~LFYJA>te6e<;bHpKL>gA!4&r~^-wi5 za%}Y1nA-SA)#y=(#rQ}~dHd?}D~_@eYT&E^JH82mG<%T>h&NUL8cEzOS9%sW%$tVZ zkK=5(U;Cf7wt%M3)H}Yfo2<@J77zQ{;r+fR3(J(s;1tO?HW5wc^YW}OX`^T7ok#U9 zlrfa)ZACCeH3leTcl^1-E2uSI)Hc!s@qmbbA=J5mSK}Sva4&XBgW!7HDK$C<9 zHX@B!$%$h!_8ZF?>XhIuvkDA(C!i}M{)ItT`juE&@=&-sGV4;laXc6vF%k zqZ&Q41`xyJlGn_P1mm1eZ6ey`;_<*_++<;wFU?!0^&}wWzXCSqKcj;Q~-SzV2>2oo_Z%c8fvp)=99xs_0ym9ko8rKP(i z)m|Hll?aR3$!NIDcEL2}Af9=_vPCPY5e#duPeP{U+RF0LLZ^AFnQ9r(dn8N!xjhj} z<0?A-_iaQ5Ajm5anXJglsLlJ|d-qao>mF_!fwglQFTLn@m;|HGS*jYTJU`gq-180j zX`Al*)lsZ6QKKx^ajKoD2T(klyC=|Gk)8B7fgVD=#xN8YF8{%mRKg}2Y}V85Iw|6z zLDREoc9UADDY@>-&?8gH4Lo4|g$eCa{oY|kH@MdwPZ11Ng`DNMhEGr!sS#79`IPnG zPt7z?(?9(S_obP;ASHJqh;3#)W(JZH zmYo8W{V?=fXLze(*?J$2Gpo zFK}&T6d~)C+h@;a{s~h&Tyx-_&Z1$bWhPnA$kk0Oz_(9HoVeWzo_!!2qvqycN%v!wSU%@KGhkuusGi#4i7fB^0i6OHC`x!9e@G$sjL!YhX&|zHkM%;`4Th` z!(yu8i){aJ)8FPd;YZOlQrpE&#ips6cM;hufUpHwk@$ToI$2%7WNue-=+si=GEr< z7|hN>NvE8IHpNWrje~hrlgdn^Cp=EU^lmEf2ckhHp-0+w-8r0~q;ptcN-*fkokYqK z3P6;pZn2Yc$R2RSBjqMhasMlMEeH9vaUOh0vDx6$)DuTT;uL{!rBrb#?1$qbz1{Z; zT%~TDsWRbNSR_WbG>^5fw>EI9r*T!r&0Hnup;QBr~oM5+Hw3j$~vz2iuhRFV%lmiKMie7udq5 zMpA(D60%o`G+Hk3XR%T>Mr1n={Wn&O98z0M83NOJN!>Jm%YmWPaV7+fsMqjCV>b-Bb@P3PLlu{PM5vJ>6HjhcHopus&UZL4RA<&)M_AieXfg zn7fUQYYMRoPg7oO#aXC@m7!S<3wu5`BX6_8ZDVFret&c~rS4U}ps-391S_JK4Y=Ub zjKuARhdCK&Jz;-iQ27sH@#q}32~+?sWr+4GGSFp)LUV;eLE>~m^jqwHZ=tq1t4?($ zCnKQpZs?Smgt(yvd$8?LXGMMJ$6=<1Le$M0_PVyyqy}qu^M@-bgG$Hgd2jR{djLi^1gHOAK}O13W}Eht(f6e-WyS&fn6Q+n<~F>(d?M z_iJu=*uc)jp~s7$qNh=aBOt?#F{T_58yRFN1QwI)l2^)O5Vgl@%1I$zw`!{TaV)1i zT|>#idTz=vzOC?os9=JCdasYknqgv+74AcvB%obi+gVX7bp@bEpesW@Q+Eb&C4pbs z%@2-T+L?U~uLBf%b^QUinl4K(KkHf?V=i>2kxp-Q2%N?fLOoNg@Q}Mejsf_b>9tI; zFoR3^j}xe?WMUa3kD6}GMoE$_B)*@pPDE!|+vT&rn8|O)lNuzYs)Lc6L3Vx$((XTc zf^;!W*#0BZgU}KXRfeRaFvF49;Jd-DVv1o;rFb-!n7EZ1M)=n;;;V>g<9MN&SP?@7 zES6Z?DtvImt`-}3Oj&A8?TLyl-nPJMqRBFI%CnGshq_r|>jUU!%1A)*z9VkW@0sYV z-8H+_jppFpeARG^Z312O$NTvnENj-u?bENIu04exD zR6m{&y388n*%u{$G&R8P4qTPAj!^add~C})(yMxx$7PkNIxg(or_q7pe_Wk|)I?{= z;BiQyZ91=vrh!DRh9oLOnaB`@h8&TczJD1)Jsr3yzkryaHdAYi2 z=k6u@#y@Loo=2Ld-f4#BVcd2{yZ37S(o?*OU>0EnZg9?lG&Xff7ioXx0+}lK6gLrx z&@8braz2K;Ym&E^oJ>pZRg~$hwF-kv0&d}qZ)<57;NN#2BHnGws;ZO!J ziD1~=P&b!_IvSwdcobvS}Hvq{&}NKRiVTES4Wqx;WqDg9gPActstsc zv2pm7O4l~=@ODfRX)=EY@bF;64>D5LGLpU;|Kkx&$Yn%6jR&CUX$<7`}N+t4Inu@+ApLsdN0tcMq^0h^_O+F*mlZiMVD73m1 z+vvN+D?a)ie_#@(M#lD*J?{Tc3?z-}r9O=uqf}Bi|2-(Joj<~M^n8Jytl~$C#rOOF z0|R@Nt#P&*w`dk7@y#(M5q0XMD^Y4~LLPMm6-K%k>Tn>^>|H-EpmaNJ6YqG^|ETAH{DFJF zaT;MR(|>bPJeU^@(#i=m{DTbs{d1~07uii99f{JRyOmU~=7S!!M$Z0(`Ts!zb1x`u zh6NZLlB48eB$JowQ_}S9dxZDuBCPfp*B=#gQgccUb@K}c7Y7e%ffSP=@t5D$w)|3^ zz)5wFHXzpgtgTplxzkr7Dx^w`?I7&&C!kVsDFbyKS?_cyJ;b9OMdpU+ZBQ5s8{GmHeB$#ZYP+;OCf!p_QMNQk@gSLKyZHO-M{iito6& zz~oq2VGW{KddW*0AhicX4^7{uA(S+xl5KBwV?4si=XRo`1BeABuU@}z z^$zfYK|7?t(sbTA3GJ7Ei^&moAJ~csw-I)6Wc8dPmh^$uP`-tmtUcn{X-T zW9Be_n$oD(mv(lgOPRp1>|Kpql<0J3k1g00$5V20=O+9opb>E|h_{wKYojck472_t z2f&tkU%3mY%;BMnG2mD4NsU{7txb4M>Qd6Uw6^HmR1B}R$KFq;^Bf)D$Y|~@w7b0d zRe8M8B=)M-a{qxuyMTJmFX`?FfD7?%&2ie(Vx8oZ>b+v%X0Kc*diS^h=`N@?I#8e~ z--nogQKThRB@BhR4xA(@j+*a7LrL0n7pv%}K^SIBL49?J!w)cx+%Pid%~sWd*4+kM7pM+w8z zSl>p%FJlvwv2jaQlf2bE!<-E6s9%~BADPzT{hsoWUc~s^RuvedwRh)b;(t9L`2YJR zI9->Mh=g`>`?!5x8g*wN;-WTJBf!-GJD5@0F0}m)ne#!U`aj@ccAV!%mZrJ(tta0k za}t()yzf{(m1z}5?=tVTB#WUg76moT~wl}uQNSDvRy{$jOp zB{Ha9t((i|rUN$>y{QwQ@`|Tfn@&($ z<5c>YFr@QhC{2cn|Zt`pP>To}=5~ zC0dmPyX~sdkIBgH*L*R=OcK*cKnjI#uHJD@yjMNtT6k)3%46y~fZ&;v;98D*=yP7T zowOEH-7+AB5+Ik-C?0$*whYD9Wbe4a0N zEHO(i6p*-@QYV$u7U0Ao={tdB#4F@ zDuEd|#^F<{V5K*@!h^fzY33*BwlVu9Z1)IB+~Em=wC_La)|{0;u(_JU!j%_qcY~@D zE&Uz8(|TKgo*%L?8^yzvTZ=gU_<}3~vt8u07_daI#Q~=3YP&f^sa= zM*Gae0Tx{LRlD2+lks9gCI2ogfh8TTZk3;(4*B8rf~wczWCYjGxGTVKm+3hB>su92Z_IVCz!o1E&m4^ zeE9WaCer+U!}K+XyXT5~;~4S^H;epy$0i{QAG>GT-RtL^ZTC>HndN&&B^b`{a5p%V zbRLH!IPCihj}L>zJzoNJ@#XD*Kz(jK( z#ZCB&Je;PZfc7kB*UMxzJbCX3-&(K^^3Ry5+Pe&D5~`2nL~(;bBL}9brP(U}+K|;7 z$%FSW`w(987O7Jvt#ymLP)SE}j1Y?H9BozW{3(hu7Dm~eG*$L7D;sfmPS_cb-ni%a zlak&f`;T}lfo;ZW+e5leIr?Nn%2X?T5N;h4Hl&u^5+GM2=sIGD07#w35dLE2uTC5ediSZ~ac5SP{PJ&%< z-4(V-n!@!0AVeH&464ifoC)bWIf)_YZMlGuLV+Fu(^fRYkLjdq)Y;z?w9@@rc2k}k zy1~29oK-a{BI5g%?q~&j`QO(Mx!*Kzx#ZS(1R6-Nygxn0a`u>oCCFd-sW0JpE36RP z3JaLsLK84i^KsN)e+xKmG(UMz7p8wV2{C>=(b>RN)QH?`t&r*97~aeW!|d&ibpT8d6{lQe5dxW5!2 zPGV$oAO%@QET5Nts+YNe?=%0&rrO<>PFHP4Q;?jUL?Z!PnaDbJ5kb~UQ)42m%1l&} zJ3FS)O4`nKc5aF3a9p^7w}tQc&DX7rs-&39`?+{006*TBp!%Z%ksA299gSXYr|;-H zBZO^;9u=*2dN<)WC2d2LQKmx@#mAbnTS@o2E?M7%#|6_ld@bMd=348#uI(U(V(B29 zm(-uY^N68n{KHAxIuZ^(a_OzP>^bt~V=v$N9+i_DEEZZUD7&6(UW#OQ-E&i_8*XVa z2NOYs511PG(>C;nIGs@t8Dl+S%|Bhbzd5`()_1QZ6-OSqV4d4rTvM@Y<6l$>GBGaW zs?7{Y-TJqEUBg~;s}o@AM{7@cHCQ}xd1nFtS)6EzOaHFS#Bw-ongKY9ue~46sm?Z` z;bd!Wu7{c?N# ztO6`k#i7Fk^R>)_*g^1A`dJ|Kg;H@m)xugA!3V=dxQ5h56E4;TcvXvTv)mmR`dY76 z@Av|o3BY;N7K0K??j4_GPA^7N&Rvjo18G@0Z3-u?DCLxrDG@yh=VB;c=B|_*x1N8r zRKf8=MfU?W{r@}kw{w(J(e?62DDQZtqW3+^JX|dAcrFA)$>bj{-mv`F`+X6VD($osyLdiV4oTJN`V6=UI2__YF8FEaUe)4~<0roB*Uhr?gUMD@yMk{3#-0p>3 zmB{hG&aEh_?OGjqi8>wrN_S=dhQmV!)WS3SgoCxL&TBwJJjytre}VZ-iXeMHg#bhD zP5D%;R4ZC*U$zi)P;DC6$D>Dwcr>ZL2KN(`W^(4Tak|IK1_8=VhIumF*T%N}SH^* zgzn8UZGu!6wSf+@)2)$iDUi3GRnOE$ewGk9QxYlSNSpD#UfZNqZ)?ya5c199>;*z4 zHgQfi1CAbyME2atMEyj!48z<=+a5x<^?`qm)Bmi_q$G08J~VroFaCbX5%}N?uX}dK z?2MVs2e(RF98Icv4W~7YWogt}QK-VsPTWhx)0|yIm7;yAY^W{TB3U-)ifQPE6Z$`E zK#+~MGS!&425Q*vyqmu%!B{yi9+)R_`wmI7jft}j38O32YF_Fg^z*hbw)f4;^K-SZ zfS>^I|9NniAUL}xFu?tN`3HE5-@_vy!274$T>0$y!;mHP%)pMrT}1aOwS7f>zestcPi^6H)kCLhpyL#n;2)mbBZH`rXy zv?%zKd@lcGy1iDdUfOdn=eC2-64?RQ5Ixn!*_nty-W6lEk=|NlyWR*wMW|}5aMo*W zQBn%~PaDuUJoSthP%`HvX2QgRqS@=m8zz_r8MAfEV9MB12<#POn+!tI6mDfte2z$O zPFfxoWOm)F5`Uu5c1pg(DbQrSPejop+26`EbL|A&TY)X>Cz-c94t`&cp#YF2{-m|C zi`N7%U8mVC3-)bjG=aigqZ<*+07TzPZ`(e$>m%Ic)Opm`Xi)UqP=EA`>`LS`L$rJu zC#}n^TGry&Q5ZQZPp4l0Eoj15CY+9yvZ7dH8%>ePJVzTnv__+I_p@PWviDZ<5rd&7 zAf(`xYl%y@RWV=EFtmls=`lU(V>)&`o#<~0j-(1IDC|Go+yLbztr|fq)Z)0?3bl9l zvHK*YFroKwKj#d)~={v z`#_lQPkL5u);}+*g&jnN6y{1X^9r=5TboD#J+2GD4+$=Z%7?!0QczRc=xwwWI z&yrtp4mxWguDFzQtnJW<(^SdU zveT6<+}5r}?47@(UnS`J?jVu14Cyx$5S+@9_k)DQbGU>EB`M1sr2?vhta?^F7;Mji zMC^|QX#7N2$Xg^XaB=zg8^hD+7FG2)IJmRXm~H9rDeSEF6|Ut4##O=B);uTdJ^rd+ zw@O=pBa8M_pSOhnsgb5)F{5WAVTFUaoW+B2D@8l`Qz!#1p;dMkbbqr574JQ>$g&J*{=5e4Cw;Ag~azAMz*J9{2KR z&|fGIZ+N#XYWX=lHfb*YV7#A~@7LwQ03%=CpqI~EVpnQLw{k{%Q_|}5HbK@jZ6#Yc<-rx^oX1r~SJ;INnFcx*gozvn=*whP;?tv;WHzkr_;$+T z&T+V(>1aj%tJwa4Nn$2yO~sY74p>y33@JdPW^|{|EgH>6`zykgHkX<`@4RNUA9Osi zsi?j~+utaYnIP;_8p&Wk&c5lLJHMlZlKKse&zm|O%dM}JSem3 z=jb(B&hWe=HI-A%AiW8@C(sSFrhZO8)7O|(Q86Nfo+BWq86*+wv9_q-tCovM2T&B5 zOk#f#HZMDYnYrBM*{;S^{hOJOp{80*L2?jm8~w9@ac6Lq1wuBS_N`Es5&PD#G>DZ) z#LUk}jcOx@uBv-R4)ZLm(*rg?S!6bmrdEuoTrL|4Gv--MtA@PSh}J#>>hQE?fI_-z z3G?1X2~vm_45|07u_%*{Ju(pf9Aru(4tl`%wm3A0L56-uDL67|F-+Pk9liF`nheMIBcH7*QAg+59=^C94*y`;qrMz&V|4SxNpyZbPYmg0b2@X!Gz9a zo^y?I>`%`)Hrs_!cJT~7Wp~z}Y7x_ufq}OQ1PLOgk_AH9cVei}&re7)%V@MvIA*(a z32(@>Ef(ss}IEoyAn)zCCwPd*1)56m6muTp%tgca!g~sP9=Pi3<+SyO-;A4aHoE62w46-Z9X#;@ zpe@to?CcNloWYuHeFsJt6{IqH&@jt_n zmTQ@t#|azSYog*DDi@ zPZjK+o*r^?BDvX2GiH!md_O{T=Tq5p=Iir8o@KDfCCP3awP~j z82PslX7~>2H{?xR9DI02@3`R>PgC2JxW$nyacV`vE-<6?$Oj-afhLZ$fu+*P?lD*x zK@A>2!byU?z7 z6@%AGDdGJzz48%v(s2P z(qoC;qA+YmiRK&AU3^V&y()nzDB+ysocgqj8>A=+>5^lKHZHuatD0EU zHa2|itD27ZR8sLg%BXqt=2Xqf`@>aIb1NrjXIa!XGOMm(`Tk4ee9NeLR8n()+{>tW z{+}Dg{(?$s?hfF8yLgsS^MK6REA(VmUAz2mU{=-ae~bS+m3tQ}##-8w;jbR7Y6RSm zJ+uBqkv2WVYlKn6{H1(@aPXhD8xi9V(0R#^HT*xD-s77ox&uixnSTu7g!KD=t9Gfr zv-*b_v2ykbF~!f9ych1UuQQM9Mqqi?q{vqoO+n;Eo63Y4%Y7W-4$NWxf+m7B!?Q`F zVpPWCgqPo)LXRyFKg{gW-7 zN3;lBWznUTl0!F>iZY+ppIm8eT!! z?P)}(U5i}fDQ;8Y4IJinz34&n0iGtJ=7jFB*Y&!=Ab9&*D zS6+zaTGHMTA6x3Im%@&|qYVRxa0L*P&;O1x-v}aKOMi^jt$aIl)tYSDd5giW>#G}i zK~(iys+Fy)Sz8UIc9)|(#odxIO+aXen7Q>Rj)NYO^;ZIW+`0h_xq)O``u;pm2^=sH zHIe#zK|fSnf?^@xt2!He(NgHq|qrb ziWIFb!yh1Xw`{{HE4vL|-1_!!kJn6Y(6|koM9;PMIjE<_&Jd>1*4UR$|oyd4a@8B)p1%z$fC3N?+SEWgMoKxRnYCI)ozyA5b0cn;n zQcNTdF2*O>WipT354kP$F_Bf2u?{gVh`}ZUqe0UMOZA7Kfel^wGi)R4hCm=M%Qk5q z3$Bk1d%7+HCCv*2goW*1HfLrf5F`@KSdAyFt&8i&3WM)l(Qo~CnSbe&;id3zJ9wJc zuLKr1<0I)XR-2yXUIQJ`H#sZzA2)$;9mCSw&e2*%YeT5yVT^EZt~t(iWpo#`8kz>S z(;J2F6m62P!8^krc-W2xR?STyARtmO?$d42t}6aPZLpB-MYzJ!8c}Wu7$Ms_&+e5A zcjbnzG~)oP?oDz`<4(f4)49AK>>4z$=27jvl^V3U8U2rw1|Ny%8#+5(gsOC&28Dmg zN`JIes%RMRBnXkQDi^2t_-=*g<1p7@3U{94T zrRW`OOqDE$M&KAq$1~?ahYT^Wp01XD+Kcbx1p?wZQ{F-=Yu&z5okH8clssa^z%vr^ z$lOdAuRwSgVke%7>sglLao`Y}mH zr%@xbZd*>B+w4@9l!U&TrhcTzme?iq@IyeVvoZWa%|eCH{RE7@v81PyhA(X(7?DSU=)ARt>HbQJWmFag|eN zQ4$ny9gD{gmIonQeOF~=U}mQf=NII(4)Cz}>tmwIONHvm|NDsWRGwiqY1duy3^5y! z=(`)R@Y3gOB-b{hYv_`}L4^0TUv{2{vMBbjPzE_f@X9N3-d0CZX~Mh9|FKO06@vs6 ziIIV=NR9pMRezOnWhk{`iI|x(8swH$W|5T!!NIa#{chWc_OjCl$b@|F}|Pnt46Z)+9#m-aRF{q6B0r4)5l9F#YVGdH1-2 z^&Wl+Ns8Ms#N6q%6dT>cf?xvPS8h~hqjaO+#RB<#{_ly7&K*KPr20EUs_|r!4t1Ia zM{x}Ye<`a{Lmx>};}v&7BIo;|s-paky5M=8&=N-+L>65-aML9Sta)XB6YKfBpkdEV z@uebFR-@D+htoA@|E=6!aw00mE&_sG1*7;n$r78fP#eSY9qRXA3U24RFT-@~J<;s= zkCqZ8)5BP~>i|EO*DFMlvyx>Qw@cT7Ag<2-C)oQD{TA+_FBb*@rralr)Mik6v^Fvi zk+RIipOT7$`He8r-UigL2~RR1xz7TgNof+=D`aSC_90tk<<>AmU#+Csp<14UpXad0 zX-aq7n|CTMYST*$;x+=8Mhl4`MX{8WfRC^SlKC?e67W?bub}xF&2B=z;(u-lHo|S= z&F@cV?D&TpU+?K|k8XcH4MFrRsy3P?=`;Mc^c(1rX5!CrQaE{D7v}Ymn9VR~LCh@N zi#aPevT_kNbs*5<r6yNxCf)3djTk0RAg8;QtJSnt&9=80gPQdW)>ccqc4rDU82ZlSfT()>PPlEBfQ z^C2juT%FL$AJ$C(AO^G!!JTu{eRHMz+G&#j)r9&DS<$t)AiCaAFd~`!qF`GBiu#t- zgQPQA%PfpH9hg4kMK$duw#q^%76yL9MQ%yE*rNyXx`Nb87{r6FRQnTe3$u%xEqLwRm9A-h(<`9?Ug9`t3U5bBxz=E$~{G9g5P<>wRgNGZBoj zRknvu{rw)m)0zJWrhtGTu4G33>9K6Meae#blm;W-*YJ))sA^p!P4)H_V64_MrG;!@#3Ec0@ZUsiS$tI1#KpN}T z(IrUDHJTJXTDQmPopU?Y~4sYnU^hs^U*q>KKy@u64%3IPq;%{9>i4F03`%1OycYEUUV0w)%ZR?^fhX_7lkOfU& z9`h1gE*7LKe#Wbp8guUHg9Vwc z=Qh?|v}ai>n~s3t8Kx_p*q!>YdZ)X6Hv{lSXu`+`rZ$p&1-`OCVimJO?m;ZK8TM82 zHKhrGYULmM3T1|n9`$-o`%{bx~o%>xS zCuL!?VcCWtxHAqp{^~W`1yVvK!wFBC?PJVz9XC#yk7hC*yW-n7^3c_&G=S=;^6SX< zh#kj6;B@QNVw&VJz2Ej0ARUw*`!t+8WKHL*ZmVJDH&&V{G_)@Ya|YlU^(NL|Ugb#I z>#TslqxEd#u0h3Y{)HiJQ9|0C-H8G+t#1-%&3E-*6(mnTCF@9Q$9l8tc&^RKkZ0#} zn!#T!PoWkd81?y(nF%eui=MabOgLDld@UjOuVtvhO#LaVR&wdnns?w?(tzm~_1cz5 zX>E4K5NmODSSHn%OcDp2yo`!gmbug5#6nSC#1Ri{Yj|5iL>1QM)(f+elY^{erZu<4_o$P{=SzhU3$snHax&>nh$`R{?8;A=u|#0U*+ceF4iFw1#qde8eJi~DMfO3RPyVJMQRbk^umVG4WR1G zl_h@4W%v*p_%{Ul5Pkrvrz-H4HTJt_+@H5!rfV$rgXSXeWC z7EqR(=w6fat5AeK`;OZ5B;c5iqPHa4eQs%2lPIXO}{_h@I}h7R^>Z1W#k4naaQ%iw`x3FgRN%12NA|X zs-zj<%j;7`#OO7Z-RQA{d5%H<95W37!%bHu2tR<$b?kDQprN8XK@+&{g_8A>Rp)b# z?7qg~Y=>?Qiaf#1G{0Y#*S+_c-#R_&*%_3GCj`d)Fc>Cgbt!cASO%4fxv$jc{D`E@ zW@wfh5O;alQarl0vEckk8t1(eVQvs??a;&i>w92}>#7Da9N#)Bclo!(PWJ!#tOJxe z#!KvCZub5w@UOth?nFVC@!I#pi3A`fwmiJ94R!vSr^{-NtqUEi(M_8qnaW_D$4gzE zKp}}g)jg?=E4ynzwC!$T9H-HRQCuL>q%3T6w_2S$NBlve(!yecxwpB>GGkxtv?hgU zw{dK^!otKfRC0oPabl@b-Lv>oYrRVz@e!<0_95%Jhs%5w2YGE_z_6*AJ8+PRy?GH5 zY!WQ4ahukR`M`eWe`eCx@ao(8Z0MFx=4ZSBN#@@dNammal>Gq8d=6~<&VCM*B3RGo z-`ANP^TKb4P_17&I8fFpUm_AW#kq39bYOb^BoXJr+oTVJFM4z0Zqb9)gAZ<$52wQr zftYBN568z4fjD834~NGPaW@#_#B92BMw7mjRtvPsd;P74GQMmxH;oCdUYQOr6TtwJ zQT%nti(1XGj)Zg1^BeJOE_Lr&VYiug?e>2iZlbMy)=v8>7dpOt^%Qs!Ac``&(LPiD zd>zJtxzH>od)^)38Hk(v<@AR(;Hcz19pRS3Y5?_|9T$|kp&8ZSFwoCUs0LHxWy)py zO)xK*tCx@2b{6xml6K3|cdf8>Y&Op;F-@Qq{jA%I>l9H5M$4x|1h0CQA9n4lHhKV0 zdfqMt;io?5BjNpwycEIA>( z@OO5Khmya&b_ul*02HVG`kpvlJ^(gA$-m9Fuyo^ar+tGXTRVu59^#hZqx~pS)kZ9g zD5z1kej^89U*3P&o&5ax9nz^O6;*<`Or$*1{=7_!^TsWvX6XZ*R&hbullnG^Z=5|} zR_CYqZD2oV@G+Smqc?Z*>Oqn|=iI@N#PrkdPP&&^@oJHO zCq3Gxyj-|-6EXFDy3RrfMwI`;>xq8GEh2;2q`FEkPBRn3eEzkTE!=l=c7)?QUJ-MF zlr0!Y$6TWhTllXbmpQ8Ssnz6kGFA?vHMBsskDM~;ptJ80Mn|U>GcNtuIGo+Y{5DmR zO&-~GMkJ9krRQATh}d&y^n0BC?Baf`QiA|ja~6KV=$9tg9IBe%g&88j@Jl{fSz{Q$ zImnIS{Vqdk?(xQVdBfLV1M2H?2!<9Mm1%K{3QAnF1asds>LNh8Xt!&x7^H!CN<6h; zP>`jZ=@GrY!93d-F3=T>0Sp&e4C3l8ok zbgXQP8((EcKbyobR4a9~{#G}KYuqLa4UVG7{sFgCe=D5&;dOB#2YEIP@LT_q13%ww z!=C`sqf!rZyFu@F%)gG^;|xOSZ=4&kC-+-oE864jhk^;+*Y8YOqh$^`h`ysT{p=&` zqLPe-96D57TW^N}XMFm`rzx(Q7aqI{~P#yBX^Xb)5w>)ukt*_s`EyS+`M!Yz2!|% zpSu@0Rb{<<$0wus5&ZrA`1s|y4C{Lb-_w#We7*E^pxq<_aGSB1oZQ{AmR$y*E?L*V z$xK%VG%#!>bN5PG2(wVBpW~Uk6ezpW98Ma;&rR%M7Pc>w4KE+h2owmS1w_3qK9fMU z8gC?#F5?57-!j@*QQqDh`G1E4&;-UyaHF`)lSOTF+iMlRh!H7`B2a;*(?uJdD4hq` zZiESLj%ny*x-MN!#~`K45Eg_BoHx2@7y(_!QcFA$bEteTL|FiL&{J?UX5tS47qqN{ z9H^Pe?1QadXH9EQnY4z~&{=lhVjTR;@rkTXq3h&5brUO#pah0FWxd5X&mIfe;;HYea z`!0S2&B6S%BZFBkYf!g_z1Xxs7v9XyNUd&R_N z4eteKl(l_I*Rtm9+qgGi5WqLbJ3N1oM>k7JT+-yb>OkeM`zP(CY7`O5t5 zkeA}WW7^l65Z#(HXMd&35W^z*-~?14gMIl?c2QSEZ3fnhL8C)Wg-sEYe@!0!Wq;!R zv3?;cS$O9xrNSE#{%ZM4eKpxPHoL{wwb>>c;?qyt%QiEISQ-Oc=f1=Eg75ixu#Art zk>gq+{XGNV>mNX9)UG>EnLU(a3$j}jj~H=oz0NE8Nr}P)eSfsYa5&)WA5akN{=e9^ z#~x1rE?MAh+s3qQ+qSJ~o72X>ZQHhO+qP|6H_y#xvzzRz^A%3&S9MkO*h;vQrl!N3 zT@`rI^9-KuWoW_v=kV;08EMoGa#%oQgkR<|i>Cpqh0;XlbYDc1`eFBR&kRm`^8zF`&#U(8Hf<{ycHs(DrmEY#i0(kDp zQ_QKF-E}LeKrVHRxq9Dv+}xk!Hlaf>mdU<^S>P-e5ezaOot z$ut?$#_}jrM?9&dnB0;oasVdB!`)xkq6No-w)CH0;drwyy~N*uc7+zq%D)|3aY?2w ziqcV2N}ZoTY+;AB(XG!EoA{4bDvx*=Mh-8IcxbO|@gR&BcAC(3Ch92-zin|$EPerA z2U*FOCV1}lt-V+HH+8^cHe7Dtk&5`4{WfS3@Tc5%&=$^cAD+|w}ieeZCIW1H<6y% zZEsU6&?P6`Z-Q)IgU$r@Xm3E)%INMh*u6q^O$@oe>vg*DY5Y_Wt6;%$U7qId&)lO- z2Mvy5X^=27DNz@S{NU(R!+8AAAv?TF^8{#N4y>uX<_#Cwvn|iSyTSO@ zVs_2P@}uq8Ad|$sH4=R8L2!-UKBEtR3cv3KY;$j{p}3TN>!6&`h^R~0c)npj5du2P zY6NoI_h50YRYX99BMH1!8RSHDhF^OJoYC zyh^yaf7D?gRAw?9mK+UqXqxV&tYuuseDUI z@HUAd28l78cZ)I1ks3++uz(c3EcxuyZ!F$2jGez@12}IDN5L6L$?e>!fb{@da|wY^ zUalYC!=;O<39Q+{;~lT*vMb%TdA%#z-J6j^O zUqi4cvey=0ScVBcXYHdnhKC6jmE3nv zRco_Rua2oC?Puaw>G6y?bp)SqXKxQc^fa+=8GXXr(sIxuT;cL5u_GGGS@CcRTg*Od z0gTX9iGAGLP>Frga)>0m4U`SMqmpXp%gzwgoaC7c1t$sap^tQW$(z}VO zMcUnNg=sJ&$kfK^YdnDCDevtnHFN`X##0Pl#kBUcg{0W*B=j1mMN08$Ont>0eW%$; z8tMPp57gWbL+v7)q;>fuw%!^&a(H}tZh96Gp}TmvVz%4nta$rb4-(GVLo9ZuNUusZ zeIB~^$7{r6o20TU%@VL;0!Q@WR9bS4~Y zH1kJ?rGE*8!)o=D8{EHVBg|R??6E3;x9x}oU0dUpSa1uk&@sRKS-5re7JHL| z)8i49?DbgG!SvtBzofM%=4fgMYOBh%Po3VhGvj+zntgG-VddrJMdXCvpIZA@{k4aP z0(6HY1N4WMK)b`*pi@d$ zXg@9-{jhQA_K#Pb;yWqaBQ1Zn$W2`L0F9g3Bh31OeNC)H4WM*2I`q#J96(!-sYT0U zsdn6pp16n!&{Bg0+&VaDOcr=LV7hiS0)VU@8 zoQqP;_U8_ae&H}zPoQ{u&R#!dtwg-DTfF8wrcK(}H%kfOJEkWjHwAGn>YVAaCC~%` zMB6I?Bc&?{Unm$B$*ExGMq^LYUT(kuQPpN|=Nt#`wE6hDJm2pf_AlSJG^}C4@UQc` zo=1aD+V@`UrRtz+9I~`A5WQxRTSqd0s;K`(Q9#C5Z@!n~4NYUeDbtVN%DVe=NJ|zO z{f*8ruc4GfHIG6 zX3jCe*)%2yb;~JkwZ;s?E-80r_+R986=H=|i6}|If{-7U+}HlOw|K{qVg( zte@NO+xGqa{5;}r*Z0T6*K_RG+bN-FvcX}mFaJkC_YYIT*!{i1C)mS^i*vqtL)Jc& zK9b(c-hH!AgZ-U;zW}`^F8*Yw;zXphSF6gd^566>Hb|Qyjn?b>vGP0`_DbALHn<0= z)?YRUs4X?_>t+MzS|10H=@q+1ECZg->QtQ%l!X1vxF0>0(}=MxN}m=^$H9_M7xtR$ zQ5e~(7g_&n*vqg+9ui-tY}+n4=O~=E<8F0px@gjY$3LLiC@(Kl%>( z8VzHCFaxcnilGa2fO_Z0drwHq6T}`|(t(x5@z41Iz0Wd02RlZv7z^XPlX;VL9y_LG z(oCQ)V@_ck)N{3^g57>)ZUUUf{ET$eD_3&A@FV}T%jB5D&{D zp2f+w(b|sDrPMl;H`nhy^j$@hx!eapZ+pw5nCHp z09N%$-^FQAB--P=qd;GI|2+T|mLkgR+n4*ZB^pzNx*RREgNYNjn&FRn=XiVD$o=;$J=2mI98zo>EA5Bl(gnVrbZ#V%52I=#I$j4gBkrLbs@XtL`rN&28Z@ez@YNiDC$8gwY#Jk~Tkb3>ixKC39!1!F z4mRZB0Hv39EMjhx5k#JF#;_8&+!}aYXU$hDel24c;*3*NAKRtV{yZs_hGL`(3q>hN zB0M+wWWXQ8PI`*CIk?6Ax@0acvVw_*oYu~jzAYiedZpE0oyEYki~4};xs^+95nA#g zpHEcTAE!HX9g}(;I`aZQ6_KdKSAwCo+#avxR6ZQSCm-%P{42NbNc}}%Ewo`EQx$Lp zD7LCmaY<-n@-*&0{_FX+P`gNh%!Hd!hdKX@j{yF%a~Uz0C9-fcu$l)K?#LYI?8c;q zi#~OVCuVdVpU!7E`ru4g9+LlF zWG~&l1X4hDV(_uC2QQbnW3j^#tE7P)hj?HBoGe#{6uD8w;p&_X*~4tX#(9F(C7D3! zIcZ^yH|ah&t4rYhPQFS)O1qZfuRumz_8E|Jp{T;Cq$*J&C#dz5r`3TcwPq&JNrLyC zMG7M%)1uM#2Hx}bWy5^cSt?7veGi1o{2YkkicMm}Nf2k&ZuElWNCajrjog0zCwDYQ zM5@^%d;P*zwtW1zcQ6G-7XTRHjBbdjQ+ZmCq1*LjhN}=Po$V27)=&0|T!lR;Gg4=V zgitj$xs#sxz|Ykn?>76rPXX8Y_-$^txYxY$125ohlI_=fk*oSpr)xMASSJ>bY|-2qb*)%47a_ap%F%>q%-bI8->MU0 zbnn_=Vz10ao@_EpJ(@mO)phQazm!+q>({{((?Wg__Qq)$L6&^N zQ8lx(K2Y%ft+EWxe@MnH@V$OaL1>?$dwlqR{ISj7-3=`nn#gE3ap`(UA%pU+78|#AEX|Z2C?SH?3$m^E+`+wChgbGRlgPI$*9L% zEoGh!#+*`_92Tu$<#=2}>R}WrWzNFZBx_~p>BbM24j1bJk5*h=b^L-;^UMd$PiE)TiH=EwO= zeGtKn?=NAM=@d1${-)A=DP=L3^ciM1P%Hi) z`LY!<10Wz_4nw0qkGe1nV^U1j?PV)WGM2}F1 zLM&~@bi5KFbKw|%wB*@cLNkIuDHJhdL4G=nCT? zvQd1j(FuN}YjmQa%YvSEqnmae>=!W`Au(QIKLnc$n(59#So#iL1iOBmvdDb}UD?U4 zjq5fmD0&}g1ok8O9Ku!!lS^X7tSmO7KP5(DM-2*bn7BMW?en|ViFVP9W=6U;9Rj3V z{vJvd??54#8b>LSsyi>heZl*UB4Z*iE?q(JZuBfUPnxY8D4L_@Y()~byU8+JZQm8{ zyQ(}YHQOw(ElI`igpEqbuLTSX6M(wxNtc_#yMAt&P%8vz&gw>}RF&NpajiMIWE$T6 z4&SC`nHZshP>9v15_`*FQk(X_&_FqL#?wyf3su`xYGl^J%dlR4xtgfn@{#(%%0(7w zQXc13Hqk&b2s@F8$``G=0SVqfw}e?{tuBQKA8-ujGMq#Nees_K{nNN>k9?}1@_^pc zP{UBqYjjgG`9iLqecF(FOUs_dU`Lns{sqpNT*X~2F)GxR*u?v4rC4ba!INq55ju{H z#8wv(i;^N-9?Z?x%dM#wAm?KMZIS-1?xY#^KL+X9x>bqAYD)=&yaIn_E29DBLWvC` zzKX%OkXNICp8)}s+Q902`2p;(H9YgqJmzV2AeORq79v9!DzoCN?W<&CGVMU%HOo_w z2H-3A`-iL=to)7Zq8=UDmaS0TL{)%&^UX@DdF_~r)87wQIGw*zTtp@`a+CU1Co7ci z3&=UEtJLviS_`uEsDnfP5^TV4e5%{U1{z3V{P3Vi<*)qk(>=GPc--(J7@a?F`fCBZ z)GLR8b-J}?j-988#M*xNE$+{@Q>PNDzg@($P7F^2)5EofyV~%&Ch;_D^H#evF?;5* zF7bRQ0xDPVn9JK9`_vLk(fuA#$F4*`V9C}svY~vHFr_bnxIA78B z6S4D)ivA>fAdX+>q(1|fIoSJ@$UGUPTCDZMNBI0j@8&QB+pEy$Zi30H^pY_ka2k2{ zD&76X3{0a9Rc_q~TVrkv=FvT!9_=;W-A``oF|doM?#Ybx8}8BQ*4U8WoxWbZ;n9`o zRo?~8(fqGfM{2iLK&PqO9Z6=&QH`#7lBG_li-0j-TJ+60xaVCZc zgD1j!K-#Z4#IPtgo|d9wH-G&+6US6KORdtDfT}MrNvnd*Kgh6-4^TP#J#w9c_R|Qw zE=Ol?!(ETpA1C*-8Z#<%+{`BPuy1ABDl&#|g?XVAZyDCkUHH=q$b>LUb1JeTo_S@nIkTh$N{l)f~EBI!LdQT#13{nN63 z)wz2U2%`f;4)5Jd$UfQ1}x>AMWZo}MMWNnQ?_{=O@wMWgxlc<`uxLhiy zE}aI5(O9F-+IC~B=CvUyk>1JeGZK28)c%e;0ev$0dM5)zA(lG7Y0XA;SNKt-YqY2| z#bLA;vMi$(39FEvIFJ7U_3R|HaGHu>YI|GQx^VP>g3kVY-sf3yHwvNlfevzAr{I{6 zz7q&IamXt3EWMgcI)Wr|Wf6hBV;W`kPlubeG%e&2*Uf$~_X!S!^ywAS-2wD73gzh1 z=M8QvWviXU3{px=G8V5245fELuyO9NxbTxX5AZJ}n2k;VsnHwjql zXf!Jz&uKd+8KbO_HDNRmS8DZ6J_HztuC;^``vW1&(CtXb#bQowr!h&{{wX5JjHmwG zw`L6YX<}44M&aXR452)OW{+8>5GM+xp67=ab0-A?eSBr?)RALUV|;3NOYArVy*qph z<626qq+>%)BprRtJWZ9JuG{4ChM}?Cp60t&3-IP7Hi~z1GBsYb8n(!4_4k2&-Nl^i zXz7QV4Ymg3CKp!$I?b?rOWc&AIU)8c8gjAtuWm}@SC{+3KGf1RboPyGF<0v5E!NtX zkCnopxvg_Pw`~HZ@;(IVZa4d98OQpUV2Fk%z8Q0lBVO#TEq zZZdM@4%>mHEQOC;B*TdXpI8SBpA&ImP?kbslmXl zCpwuX9^>!$jOG>1IoSo%4br*`FQboF$~r&YFtE$ZLk?)%d@Lz1PNmG<7IY5f;bvyR zdy1hF6{2JaZPR$r=*C{6ohS4gaEAd5hM(3X3tJ4skUz^Lqa+7hr{SEbsWVJ*L4+@} zvY;1{mwgI6x&5b^5SS`4UQl+JyrLo}iU$^PCs;bpcr}3hRd|puduGOqNq?&}+leb> z>17oZ1TVUERfy4*cuk?{yciL#WKO*7FuK9^by=KroTe{~jgv@a&l1e^HE$(^=%>~? zC;*k@&NJWz+NT|hnl=1B1(sGZ52Xqk^3K!gC5`-=>V^9`lOudaG1o21a2tE23P?#$ z(BU`_cjfcAJjWu0yXWgQOSi*K-#;F+me(<6D%vS7=q^nAI<6MzRSn;p^yG7zw-OPi zVLKDjXOUKp^3UAl20v=m-Q7em0wl@IdKYttlv?T?)Sowj38qoh>}dqsw};@4N9eKR zg1p|gyWa>63k-@fLxaa=POzS)L_=e+j0gDPN)I8)M-?+cttn7bp?>K~IR*_~j2Dw? z_WTEJ7aVR1y#3@$iN6lvkK2zpEACDy4|WzMHSNQ#KEeAKOWAV$ih?8fk+twp7W8wO zNEge_(PWF(HpH3tIq9K5HX$Mt1vT?Q||ypd1?&%3ZwYF>p)+t#~xXmeqGHN>bH z?eHvhK=O*M^M~TI^{1U9^Z{37Kd2<>)^TQ5lBVFnya59?Ayx>J+V%-DNNTt4`o}4o zQbxPjd6o*|+wD53<3I@vn1Tq0BzQAK4#XS^BB8}GrT!LN6y+7vkO5d;h$f+V&2Dc; zaXo5-UpDw${VcyzY~<0jb1ICf8*T3fdui@rri6a4k5xjrJRh8)zi$U`wW9)^eqT3l zZ)azr_B*~l9zP!vzF&`s1rxRR1_FfNefqyxQb%uZwVj-1HG)k?xUhn&>gp|K|;Ph!F@|`G09|{q?nF4+gJolF>kEQ01=6-33wST3UJtnPa zYA#@d!5JY*97M4AP`qEv+f1zA#K_P7IIDg2m;sV`*Kza^pUDcvQHW_O*1KDc_%0hi zYV?dK%y;unEtdc8b^-?FB+dq1+MZUQP?u@@OH&@uW0$tB8RxLuP<~V6+44X%DLSHQS%qgB*^i^b+=#`0JAOm2$hC=#WF2xd_%iuw)egksGxIE0C zc4#+=VC#%O9c&%|$o(iKXmlK|!LeC(S4gR9As*1pYTym&$@Gg#&L&oTyjRE-YNzNc5YL$i^XQe*x~69;Fr@~f3PqO zGRyfj-=D#%rP#Sd?reyf*=kymi_ZFr5+&{z$}+?O-ks%{ zx1r1vZp%-T2p_;~yJuIXOc4!AUy&# z8{HuqOipF6ajCFev!;~2tz}r~;oA$rJ7$KPhs->AWIJU!*V+I)+%F8{bpBu=-x73V zQ4yk*N%rz~70y-PN;^6A%OAB_;j!BGE%~Qt%9VU(lAu%*qP}FQdg=5&%igrHKbtqs@!SN4`ja5G+rK?zTC?s z7dzexL($TMX{BHt+k!8&u{v6fIY_oE1^zU0%s+> z*XQI2kRLHmKrdyg3|wbV1Tp9bM<}0O!mBc!{cqi8Qak~#XLSUpA6mZjU+cGyw$BzZ z-nB@Tz|&yPciGHGu%x+cD(Jn96hEWn+UT*bkA|X=%X=;5>__3n_o!{;$T=|fe#Nm0 z;ju<#x(iJv)+Bz?RLI9U8@f1i?|wt6rsHQlQ(dxLtq|T3IEi(SHjfiDoy7oKNJx@p z6(B|mRaK(`nNa8jh~kSNyXiUNR!4e>a;FW}LKb6hrgboX|C)kbvFo$)>@(Fn2f6$D zRR6wvQz5grtAEV)>u{igogN4u^u*+LpKPt0dfNhgBKvE8vlkrd9*l|6jBQ;D)#!)R z8){4wl+Tf@sLJS{-wm{&9banpv%$LdVZpCs@+YHr%5eHs5L_9Fg+%D zCe<~Fwv&cA$06I+I{?3FR|jD&em*nfZY#N*F_Fjl7_LgFo8Z$h!2ddQI{+0d^mWsi zibP+%+d_6z!TNQ@pAc#fB#bC61=bRv?jDAWQoODmQCbz0ZT78Z)b5j2ha9TL!g&OB zl`)pR9PO|eZi*`9Nc8Oc8IDjuLW1o0M=HVgD2q5K%RK{_G#hFt>)Lz^VvxeWtnqt@ zuV-6$ycngZwE zd}*j@KEM;%Uwl4F5cZ-rb4h_2gwH}?drJrvN6Zb&gAzl}BJ=_)$A0u*t`8S^+6LeK z39EO=T!h;$yx@V2010|){ULk-zO4Y_{IhuNo~BEY#db|_KANCR1Sw9R>V>F61Sx)K zy=ozq7!B2YnoK;}s4$_OKy0xRFYVEs_8$W?v~3U*y0y*Sf2s^&Y1y!`(uZe`%QG*w z#&WVOOwf!ZF{}&x%NS$Q+g`$6&>kK!Ob2M{P=U~2HIYTB^|Q&@hfoc+TR34!ZJZ>Q zY7>At1JS> zl{HPMoVKtkcG;bzq%hV_SD!7Y?THmpi0GZeE5s zC5o>JBgf8WHd#q5@jmq7C-2HXIz<&5nXKW4e_*wsJ3|)@^R-{$W{a>}CyQ%vd$0!* zE##~dLxxk$p6FFJCO&g!ZN?{E8w5-j?%) zn$oHbW?W+Urg0=o5XwO&aWZwjn?-rB>NpAX2juYwA;~xZ8Fj!ftGk2=q7@|)W1h9c zudq6Mm}eTeji_Jbq)IsD#*FHk<7jo88?i_D)T6XRw+Bc=XuJO3@|>cp5K{#-gK3NO zs&=#Rc8Q!hz^tJ^2wcx2Z*;hN_#W6`e)YAh>wQ%d9r9}A5y*f*S5F(Bt||>2kV>Oc zNHkTKmL|)yE-jO&yG7=bg{*9m;_AG<$krKpg|;AR2rM6w_RRQXoY&GFbkJ^qb~=OQ zOhc-wYoPFzvlcOkr&-Fe5iIUVyd~X@b>#W;N^(gRJiBZCM1tuRgkv-Wc}zsc3S{Tp zw}yJw;W%gmc2Cx=iAYorU8{Q*WKG~s@B{!~O&q1eEVG|_`)1q{IKw9T;V2hD8Mo~zR+C_A>_ET*|b5WAsB;M{^Rg zc)mj(O@pqIvn_CPkX1`U@%w4K^k1V@6w>HJPbrY3-G%6Ch6_=&V5WQjewbF*LKT`# z5FpP`sn{*pHaD+zh<(+t7w&w=xsE~V{$H*75@{d&9Ri%Y^itvG_?8QvpP)I}TGM82 z6DLO-2Pb*k37q^JelQwj-QYC0AZ(XtcQM$(WEp}MLO^~y+4Rh1Rb0Ul7oA57a6U`~ zJDsVe1V+@rDb*xwxsV5dZj;*Ncs0Zm8>wYm#o)x`A7-fbnaqszGhtl;6E2;@9*Nr_ zqsuybo)^hq9Q2QT0H^_@;vGAiNgbiFPh42s7pyp=UJ%_i%`!&9Nn%p$;l7||R8~;5KqMnQAVc7c z!8=2xJ%a!Y?T{tOy;{T4f56osc?Q4{JpLwLAYVAJFW`#$$eV`tf(9gW1cj)@gcMSq zzt9V@@pydXOrD|>k5D}Bx)qOOtfx#=JtQAEscpQ>mhVa*IGFF~@uJxAg4p=IPP=Q# z$ckFAz=_Y4NLnWQ^##xZFsv&iea3tcq$}>6RTyh({+_I8EA(?}k|$7ohOPm+!6n8} zAeESV)b1z{`?a2Z)y5EnfSwH)E+{H&qqRzW#mYENWdE*<_&DK*6`(CO#5A?A+9yH1 zVPGxru-gIM;Q-Cxw#ZOtR){|8roWQ+OnsFEpluh(f|cSR>K;yhiiq%mYJ3*+9|09^ zMr8$Q)=))Oa5&<;f@;x(jM5Stgq+LWg#dCs>xD*gKAM)=cn5`8&}UYV6H8WQFWw;) zZ({|tE*0E1Du5NxMmcB`72pi1N`=Bu1#Fr?vQz?`_$#%AMihjZ4+U$H1ehrr%qZd* za~?<*zDqeO2OF|yDMURYVZ+ySuh^Z662sxVM=ErFkNGgbR6KCPd&Qp_5hBq8PO*3v zL(vlLMvpdbp=s1l$IIKmP^!>B0S`<$$`BPi;i#WbYz_{T^jU;ZEbbOU;v$qUg6Ms{ zA!KbKOvzC46-XE+GjKGJXxQNEus?PLgm8Hgsa|RzPoiGyYQ7G`#4w82y#Q*06w0Ux zR1Qt-+2~HpBtKHT4`WyaBRW$Y#H^#J1hu!0~{&>+ZajM0vC^R zCW#k}a0wa?O{fKz`%LIysi*k-6H+A_6c`5{XbJEd-YEVvu_q88dYh9c@Cx(=O~YE) z?>kWDOA_Gw2ahrq^r6wuKlgr*pR*je!NlLk)5r_N)+O)*jK~WR+5mrs3cQM%*9Lxq z3b~BB?^e&AXOWfzG=5E+ZxNQW7xc$}7Rxscink;WX79l`HM7&eyAL#a z>VZKfGllg)^V?pmv>>X%+W41Afq5r*mlhWAR&}obzI2)+XBh;hX$M&p z@WtQ6{Ho-oQ~Ay}Lgj&JH;yu5pN4~29qK(OT~6dZFuX<(|4bz&p_OkU#j)xVJz2;I zIrC+4D8Tr`5mR(s5+A`_V(XHJXi``5XF944U76`N^wVLk^8o5N{{X<&VSJ7DJ6^IU z;!e?PG=Nvm^&YbBzB9*0P5dKcNv*71a+LWo#YHhFZsB=a)u9w3QCQDm*|{5`rPM&y zEAoZ*44BILBWUK{xsH``_l8T(Okoh0n!K<8`$rhoym;+fH^P;{XvZr3;t;sAFl2`} z{you~MJ8{3jnDxPgg4;Q$Yq+M;NWAH6smNM}eV zXJ!2Xe{AUGp7nK1oVs7heBFu>;1oIAj}2J)fqZwLo0Z`w_yU>U?-Gz!;gR@^*YP+? z-`uGrMC^FbBhC&wM;!>vuc!OWIW{>f+Ee!Tq>V}dBXLSl_AjB3@IgCTGLLLwO-#<5 zHVPm38;W{6;M)jaKGYo7T9BeA_?c|FDu>%)=ThZdW(H~xz3E7iuUwtBWDW%TUz)vL|Go<` z05ut^{NTJpox0t9|B)^Jr&fK5K8B<107$3G+6M<#`OTHqqmku9eoUpBlvwgS@x9Y& zk^n#-2R(?J9>`Vo?%W*L{Ua}L(SM;ljNa0zTHEY7kN&hwljrpxL-qF@o-u+@s(Tzo zva>SRYq=A<2VF4j^iSZ_2QQX(ogr!A@x<|{`^dz{L_M^ZI=WE|YUm4qL1ih-lhUel z`gvB`%F}oddXjNb7?Z4K@y3xYHL5+dkC)KPu)QJCz2&Old{Vv8a&b!Xxdps!)~Js( zUrX8OiKvCg&90l4JgDvn=CN4QbAbhh#1E`Y8wTTH=ztk&{i#B0F(nDBtIG-vy|Xck zW-%IM(9Ob(l>1CH%2{`_4-?GFX6+&m4Cf(i>A^A!EMh^!A~YINXzQJqX#-3&LZ zTPWO;0aq4gi^qgFby{MkG)t8%h^E`6H%*_zx0dtL0sr(kwwd@qi=$r%)zn39z8U=9 z&~>*3L>6J**eSmpIT#`Ilx?DPJ)?StD7A!T<`MU)WnI*V#919D+J|wHBR?bDWPy*l z;>m*w^D1FZ@q|c1yOp zGAlxB+ydqq?W$JkaYPJ;=+n5_HDRj5-5HxjuP{WPxgz8O=s5~cU@**GbbK`8pB!*_ znUTt)Z@Ge`d?Fj>3xUw1f4VR(GPLn3weVRpD$oI|3lh@Bb`Gy9&my)?bf$G>G_}*5 z@fc)FMJSutCx;Q*{txcZHnNw&gJReM-}u0!$?N>xvvWJupV-*-P>$&7F$ z30WbVkQrL~6HxG#va+PJrcg!igybadKxT=MmtaX-X!8Ka4``(A{W1CiI9as*FB3pc zskA>Em>ifP;d)8TknPt$3#KE*J%`p9Oe2T2WuQ*;zH>L^PolQmep^{;(y5h%2j zub2zjQ#Y;=dks6%Im?#Bpva=gq~@;Ey05+JBTfW^JG9|Y2d+Yryp)Or`2fXZ=_A?J zhBf)V7DAz<3c9~THzxWNOrnfjSz#m9x4-72E>Tp-k)ea|ehAaF*kY*Jueq))OsUnE zVU6Im<(5#wWj_I2s7o$*_8(%h6-GR(R`T@3ih^bst=I>a@>cu%J;MRuCgly~Q2Mr^ z|CAdskMKL2PlLopW{_Dk*`ReNVGw$_3@awSQPQnbbR{%Jp^qZ-)?05P_hn@!Sb7Cj zTh{HlOynlfJ8K1nTG|!-GmfR1U@_3ct{55f zy8C(IyQQ^%Gs#u_k@;%|*TZ;6a>EIPwvipR0;bDuzk)4@=6HD_IZAC%hXV8`Aps*B z7Yw&awtYqbEK6=sc_-4wBpn@7zurV6veo)JceeLy@QVuXU%y?a4gJi(K=99DD&OLw zt=THGvu{)1SJMT@q$>W$E=ms`$mmV8b&H1KbY34e)P5?PW88RC$@qqu!vRAdP^Ut3 ziKZjQgsbr|XkAcp*BI8jOm>&wK#EMt*{^7Hq155+X1OcKx51X*xB2Q0Vz|Sa)$Y0t zWGw$wws@F{s2@ttwlMpm^E`7y2| zA2z=Lxlj3?9uuizFS1>tNUT06D^O;RG(pZD406Rr1N;pI0f7WJey@}34BHG|jY!d; zW(rTKmCVtAb|SdRUu%`Ezdd^(Jm%3#4VKNRWuSIQR4xb|uDenQ{iw}mA3mgbN|fKT zApNeuh#r)}wZB^PHg=kd@K&LI7sb{BcKHrzvvwXS0SZ3+xxB=1}+ zi&tPeqxE^kCsN7t3+T?7*&m8tW&WU|-%v&_o28=!6txOhSzoq?D65bnVc!;5n-X3&S14?JE${U+*Vr=%tbz81Awqkx2V=7KyxzDdGvsw`e@AZ?=BVW5c@XX%n zo6zu{Gx?KBJf}Pw+Xd-NtOIto;9kiH4^s$Ww7lU29Z4ER+(XRPZR37v-x5;eKYlMi z-&}TE(rBZvr(ful$LdlO6*?{E30KeCay=1*;p{CEs|!XhlSbDS5fs*(7HG?D^Isq z0&}06GQ3oj-MYqlW4B%T*K7{d4qRP9C3){vw{;M8%WAZbf@^SmI)bSZ*wwPG=~XBN2)8Nnq@pbV3WZ;u|_z#=;87$ z#huIbsCx65a^w%rWg)@>dJECJk;5erd_C|b^*6jAo|+%?u5<2+lhZ@hPdt@V=b&ub zo|aP+wP!q~*~O*8D4>piWV6URm^$5$Oc!n=8dp*D09in$zcH9b7;)wAvP+tX(enTLZYf7k^&?k5CIr^hlf;?>vE)+N;E@CZY3+h=2y7#Lr@WQsH%gqr95Fi&MeU-dY@e{AG4e0MT5Whl zf-!PyW6@P0GdiMm+!dcj=>43|d4~#H3NWjtAMP~5KOyuCqaB))h5@BisEP}U3jTLL zPlOh|o80a`CO93_UEg4yJD&lb!$cd>2~_Do>k%Wp09Q1_if;(4`G0+OkkLX`^7LV4 z%wylstzF>opx1H^%hCC6#n2+_yl-aIPKNuY=6&Hgp3blg!^yJtIMhzR?XT(^Bor@% zu;%tW*K#!2hS$hZW@*Uml*v1LSjx}$A1T`s^JRyvZw{5=G85|Hf!+O=;Hz{=6>Ww^ zC|5D(CJOD$@~2Dgiu@K+-|CdwPR&Q2fxp+q-V`#Ii0^X@E+)f@y*|n{uFuMqH?EMz ztHy0F5-*vvDRL@+ZkV%zDs=bBQ}uM+xv7`t^HVtvp4pD{%wH`XBpg(zZ6(7iZ*V3G z3#GZ$I|+5k%z6z_TRtrRO34u-EPt4Xl*rUK@4~SOZ0~GuwofG(@q{X6GMbOr@d9M^ zf6HNyXk_4PMjBI?XI6noPnpAWFp}sVZa-IF=g!puc?x0FZYz$=ADarNN4MQb&(b*w zy@P(a>A${WKk`5FW9aRFe*B*O#&*YL?HjbTEjV&>*q|+i!IcH&+TdeAGlvyG#-;xD zZRe$aUHz{8V`9mcG$P{V3n0r%@T$YJJEPMsW*{1uQg}J4TwnaDNl<3(-*PO+F4j9A zUtIJsIrj{oy&!uTXqK_B!ZDK@#^7I9K%}K=L)!G}z$b-Lx6``kS>T?K&_lgt zYK=F1tM`Q&ndE8@>KJXRQHQA-^;o-zSycdT&BgjB(;H;!+59|A;K3hVn{$1=xYmY? zN0riX%7@5(ou{0V_f*5s9CY}}W<3gMDn~{MFeC!L|AMB(m4QKJf$FgQ6HK+}OhG^zWu*2ndllU@w{&H76K>%UGJ4sT{^%@~5D zCj}=7hF&DzE7%d)*#Iy%XCI$-mmdREQ#Fl1A}0DVOr=21sXpTL=0Kiton%y;F!3>A zf)U_3Z4dDF+Am6&&sdGg5s8H=&j(I*1zHO5Jh~CHsyu$x7P@;+PY%M}wi`{v%i{dv zCXHILUJuO}T}PieCpH_KQE4Gm`N1m*-~J;#2vzVEQ|pQB3AD0E7Ae{$l%ckK;AO$ z7SwJzAgi;a|Im2;k>gA2iFm;3&H5yFgzk?Ems}aQ9|2M~Y0~dW>6Kw4#@Tis-Ji5~ z61_q;5z1COe`xN?J`*`te+Tz5?x;pDqL!?*X5u7k_R@OCnTFyquDH_U1*C{0Pbr^a zC~L1ZwQgAfDO9~>UXe^RVUv)TLytcBJe%Sv>F)@BPocg!P(5&?7CvsEydj0a&Dbu* zzc8B^NM&pgQ#!AfDfK)l?U9C`TJDkr9YZ*S^v}%bsUS+UF4HO4de1w_La?Db>$p(+ zSYFUyt-=~W{PPNsjNYVSH`x~d4+=o_za2sz58V-SGu^SPA`u!k(5c{He|FywVHT&RszbOi%$v!EbTU`2c zWJK;w{k%02-7?_6a9U`Hox>VbxI-xvF|%E6jSR+>3sB%G6D3b*=1Zw{X|n+aYCG?1 z;lwU+PQI@p5JraTGhAA>u=Yl!U zmpS(q;}2DvYZKNvVe=KZ)LW=^)a*I2mZ}tV!;bnDM%Y!2d3Y18)zP%c_I5|4XIh~2 zJ{Zy%j@+Bg8EJ9y=nTL5qp8N`eKjDAHfD2Kc3E!rz0W)LsW@Z$uG-e*To7wU%{}e+ zdT6?jt)o)OlsG`r*@}|%>i=LGS`0| z9RPh`#(`<`U!ejX@Uu|a^%cK^qT?|JJ~W1|S(cN6rB$Fk2TT&U{e^E0fP4d2^zo>1 zkVn}1qyLgOm^NPmeZw+uOTzA0u_d`wm^%t2o+`yxViWT1FKt{JXppcvt85(^ZR-fE zII=;WAJQJ+8S%q*t0Bdoe0Bcx{OOK@-wV`N{IB5uYOB2c@8zgJ)%mmYrzd*N&rbE~Jsr@>{rm-i^JNKyJXLz)SM0jV#!&`jOtj>9w8rnZ5@|a42qRX3c?} ze=CFRXtqn*JD%D%7QUbf7poUiEct36{{8sXe_XwKfBE*+yPJzQ&&iGdswP8AzN!jS z;TY5Wy%_T@zUI|wU>|cI zRO)byYm#v$%A4LwI*pBa0S`L|5Br$Q!X&B{G+}B^uB+Ts&<&>HX-W6Px-9b#L>A=3 zs>qWM-2y>>r2Gt8clG>9L+sWD!nmEtjSbC;*^l~Z4B*ma)$M=>3qIQ$fOfNHNZC|# z_cY3USpCTJJ@!Vww$BDIk>-+%o?Opnrl)xn1BHx79uWaDYpz}LL|(`{p7yFVTo}t9 z`ByCi|L*y}>RsqeR0SjN8zxre`Z;;J&jQzsrmQcGWK6{3prG2$^E9DX5Q}J(IJikv z*+~tz4FHC|dVlpo-pbJuD+)nG9-|Pzt`TsH{QRM|GPT< z{b~2s4dwb}s!vGS*{GX0*Tje=rG!fe)>LK64;iN1z5GG?me9zaqAxU@C8*MbR^?h2 z`fdPegX!F{f-j+XEtKjHMX4t>s6e=OT-FW>YHw*iBEji}ea8k(LwQVEQoFx#EqzQy zjh`;}$ix@dA-oTo{vyVg(K&wSmA^5acFWOySmhln?l`+Yzn6uUcxRd&&XBvQb7*UE z{>zxIAbQ+<^YWYL?(oCf^gW9qne6Up=LA-OZ4RI|-hM9Xk9xDCi&@M%RXk8qNB?ap(euASV zE9e&nNXX|yB+RYEWqMKnS}>WTyb^&dAkko%AOq=^^;VRq$3%AivB{3P3rKSY?=p^` zJp><3G<)5!y<@tADe~et$OZ0Krj8k9K1X?s8NycE#E!!Tp z(O4_uv)Jcbw!I$UW&@n8YhA-a@!;I!_}n0mdnEq02XBj{w!=K_bLMAbd07Ofirlw2 z4mF-T0q+^ch8WHi$CZXTQkWY>aiV>=5ITRtCHNcr`>hExLH`kKuMLrXKpGZYZ~tb_ zT%^R#p8^m~$@GEvqa%alP`x*1$FmJY$WW_{X zZ`OlCniOPyQbP3E2|8%~vWq!a|O1v{0zmDOd$i~Ol?vmn*L)QV2#jloV)Eku7!iMx?Q_PlN1B# zG5xrEi?6XGxh9AP?^Va<#+B6fX73d~p=6_f(H8)!SDVgBn$Pq^_tSb&`&7J#&uQ5! zD!LqmG@Oh*Xd`UV(#sw){q_bLUqg={D1d5&3AkPcBsqTG+mUsU*5RTGnbQ^RkF06e zS8Kggmt@7j`HeyI0%Bv}sZd=fTfRz@a*eazR+$ho6F*D0gch}tXXO{WuV;)za8QQXIK9I+(xp$r3UIfinyw zGhG7Dei8Dx{2|!}iBhaoV}W+e(o{~DRhd_1L-qt+$TZ8fpQ~Hc8M&te;;tlQzCJ9| z_!3OlT#Za~kVw-kn5@q(bq#n0cJi>b(Yv)X)0IgnOeCjb>oHGr*>l+$cRE;X|6aYy zs498#hfF57dN(4|>bl-+tW=PT@2|;Au5Mk2|9)jnFTd40i4v`9W(-nTKxnX?n_X=n z_A!?#oG@$mW~^mHAzqvtO!_6gR}7cC(96|-W_|XD7!%Y|^`~BRZ8tt|^O8XagU%dq zudN*?qNyV6^j22no?-w$P?|Zp;rVm&N~j8}`;@KE4B96kKh0$^ENZRYHrx~|+e?JJ z!i2X0N7NDfSblwUMvgu%mHv+?m+ELvE;o6`6JBabMJ!oSAhs-+c_9dsFMaXkQII0| zeKd1v;W?G$UdzqwPIE-)4e5#=dZ(+IcMu}vn#c|Jx`M$a3E9S?EMt(O9nuxZ4vxB4 zESh>hG(?hM^1L1$2zt3B5=%^;nQE9=d4MbTP!Qx|Yrn23u!n?DVA1^PJY{$1Dy2`) zUy z*wbJ;&6pB8B&v)KiO?~~4szHxY{QBv zkS7u#`>VGy9U)YBFIkfPCq2U#_&eEXXyly~EF3)Wp3xNAg`VL*zb|;ng48IA1!)u? z>fGB{kbS`}#`keS;EaC2@S-rY$mD_;WPgZNeEa@#)FQrpe`(jC|0$O{(}yXB3FX5~ zf25#V%c-whGHT7rO(I-y_H;+ajg&5fP+xtVe+s$s&^^E9A{bcGyv6!yNH0(7^lMOh zuh&!)QnIPClIMX6B*db;Mlh(KYOzNvoL(&x5c#vld)x^@D4i(S|Zb#+bI zG?EWPJV7(1aIYCB5vFD;R-VqSbcjtHU>gpmMx#X@C-LmqK*@>z+d0@)Pv<~02j!kM zD|Q4X`(m$9bfEk)%!@6>4L!#r*ETh{El+LMI#m%gCDstvdA;)LbvWn3L}{pjoKrp5FH%h%-fwUe+tFJj1!0Mk}z zUp-bN<4YFkCLGoKQ3@XFM7pz@WBZ0a-S_q^$XoAMN)nl6ST!Mty>f{g;Xrn%Zl;oy zn&18lC~rnhJI4b!fB)*@<=a>DP5R5Vyq}eFRw*{)Wr#CIG)jw%vx;dimIvH>q{G}) zuJsmvV4)S%!OE7L1+wmZtMhFGbXblg3g9WQsv` z4In-=8z7$sDaUY|aKLSQuj_Z3yM5jGeMo!EP2}z^+Y;zgxRiy)&7z4*tW01N&MU== zIez+c+1w#U2a|ozdg`$BZ|!@w?AlrLv-1as2@XnN%^W`~4s>ocwwwIv_ck^MCni7T zbB^YHlz85eoMtekV!wn6eOKd=UO%l7Q2H#Yu(Do3QwdQ@1)O-8OL^!_(+wA&4K)~+ zF|Ek5xBSRO+P~{9lO{ad6d=uS#tGwm%%M`gR(g=adZC*>=+3*<-u}&2{Vki}*JpWJ z@BioHIBpu{!UiJtXhu2fi5q2_8=5Qvi*flLd~W0uNPnND-XFy`57^E)8E=0%q^8Rd z(XVW0@vymSO7><(llc4@`-<65#wOs6bh>RunFd0nblPB1S z@wdPIEs+JGBxMQTv>yxtMhL%t`gBhI-^JTE5dN5#3L81&SM>u8z=3vqdi?wKH}C3} zR?Rm-YjoS$2pkSi^EY3AJtr@D0fp7M={jR0GHuAPkR_qe-Vem2ux`7?I`1X4|HCu^ zzFL>2@rTCVj3*_QgGJIQ2b$v$8})~CaAF`V_>5%ymXW3UVhhE-OS5xQ-|63X$0jdR#xViAPiH_nJI-mZG$j3 zw7AteqM^uh$U0H3=tRLXTJk&eE^X-6&*mDO>+yVkj3h)BM3uBKcV7P=haSacdJU(A@ z$AghViQ&XAz>d&R_L>MaeYbBg5R-1k&{1!QrAsK zp)d^b zu=RkdA1keb6kY>iY|C|UnC^a|(Y4E^KN>l>CR29B_SQk!23*{A_n@3AV@(8C`A~Nm z9Nw{^iUrbl!WGa7IeNx?X{38yQ&r$SL^H1M*pstG5E@<>_bg)iSsse z*!~q0ii2Fzs<@+%VQ7Ea7e+!(U&c(G9{s0x_58mZPVv;TqJHCBk^AqEgl3s)*Tb-6 zKP;nbkf6dYET(adgmyZN`n7IHU zw40nQU%azr|0rs!86pj^rB8-RU)94hBFnJ$562smM|e?|1z%KU7-h%wGV|v!c$ST= zY|jYf8m+nm8Jn36PP}UtVWD;xcb72MLIKDF_+y}M3rtuh6?9w#dfaT%ho*($lL?}3 zM1lm&b6>GGmG|PF7U{)RZ~j1BpGQgZGDQZGR3(xos%EP0|KxH6qt@RAJ-`)*vuQ9gPsR6mjh-! z93bpaAtZ7}b%q_n)fsB>8ec)O%xS`c)wt^67;A)iIew!ERCluCY0T5@3QToX(b9Cm z-%$AHYrSth6}GwtS40bQCBtGStk@_bmuHQ*yra9P1G>8%BivHNB21|GU32V~!gHo_ zjVyKH>WEbMdejbBxYoF{kFeZ012qBhY8ics(S`QO0S!LLo^Hee`NNknqON!CK0I@l z*rxliAzH-T!eQZ~EnvhzjS@kcVnzhm826#)9E@=uMYs<=$s)=*8{Ko#U$pzsW13gSlep_jjMxOZ@ivi7|JzXt)oLkLUAa4E(4S zhN@i7{)I4+$TZeVt>;*%62z+76y-ZlH+H_G+>$uI2W=5!>lD4-Qz)uMAy@IpEZ7xE zjFyQk1Vu8w;ZQsvb&b}^3H~@w@+yXAn5Q>vBa7`>-LE~#s|H;2#Ayt@EEq|uqF|!T zwvQ-txt$wIX0J!{P-=G>m`@!~2U={>l4ACdc&^JuyH|wVe1C#Dqsb zJ`!C7fd9zeXc;Hyj2hyS@w_pbBeoL$>@;npN-06lH05NLB$ElvBFg2FrW6Cn^UB^; z>j=~+q`!#LADBfm6*hR&JkNLn1@S>sg=2PTa|1PfKk`krA(hfdE`@m}2-VyeH^F;b z%O8rPZ~*|D`umVfaP4-oHU&XOsZ}pk1ZG76r>-$q-& z+8shAOd#4Pxjvtzth1+tGtAs<#9I`1J93XV*_&4x+6#*7UCv@lsEy-t)ZaRaZ=A8l zu)SoPn940Ys#-iw12F~?L1y~@S!0j&@GaO916KNPjfMsKwMmCbS;k7pU}3IoKSs}N zT&*Sax=UzI6J7?I$`v%Z)vtoj4yJcTl2a4j>>pb*#iv{svRd`a zZ`8e=WK`MR?SyNzWJS`gpu?%V7+0sbG~OSW2chK3m0m`F(~FGq4LMPakyjR_@h{h5 zaT%?VMKe}Ry7o~5Yk_%0dC2E(3h?V4=$z!KL#xg|_oG7raagvu028V7;yNy_E|ss% z=IvTa%h>_p#YW%rjn>?dsd@RS&_ny_<^Sov2*tXwWL3DXM_N!Dbw>#z0NX5Bn%2E%(3Em7SYdC))psw>2%odL{a-<|Q1 z*f1{=CN5{wX#z6mkXGVt7@RZg5!6yD=j5G?^t56?Hi*j5WesbTxvwFyWGP82kj7)$ z(&LD>%xdXw)Q;ZNiT_C*Wea=JC2I~lFEv0WN?tHq*FYn<+2~3jVmG&Z*f6bW_KKCN z4$ee=rU|2wUX0}im>nF$AXI9M72S#f9-@+lFKO;OwD>VDI_wAcR;F?OyjJVB`sF(L z0+V|bn78nOD7V7~pp@+%w`5K4_O~?G((BB1nt>_J;B&L5F)opah(bF?dfH<0RqfC; z>N?sj)T;dB-ZiA&#BI39posQVb}+6<&~x&uhJa&)VW-h!Le7a*@#p41Z0)G#<1r3v z()u7sX5CtPjz5t2}}(GR@FE zUdb+P8Aoz zQ0~miuY}XNl8A_ou`wMGNel5{aZk6X(Z$szylnNt75op&?W3KKzAytx+b1=C-O1_>*jLz_S5Kj^QW5#Dd1^#>(F zikV5)cg`ClciRYJv?)*#C)pXH4QA_uzqq z4(pX=vf>dJspbd37Qv9Asw82NSIL+S{Z7a=3ozTWp=GjWiYOU{?u2lmN}6SLuWo|? z5+qLJOY#NLI1=t<-x~{Po>60>T2>;#P)A;F#{}u^C<PmO?c0n3?bmhLC_KyLCoIxMK=}fo#%Gs6B%}R!p#h zmf-j_6+HQp(4DC^uEO3M;iCaMneCB|q}1NA8wg-d)EHndZ{%@7VW?B`t!V-8_>0TS z?y(QV2|;K>K>GlCZj6awPEIbeeC>TXlCe12@p`!eA2|bmN#kQUE}AViF-6`@1;8FtZ7{I7QOccChufv0u?~ArevHAl6L@dG`^SO~6e0JAQOmrGXQ_g%n6<~OONlJCj@N+qkCllK(0_-F<55J+ z?d+UfGut!AUvx7FEE=|^*eNS1&s5DzZ@T$+G|&;eaLl%P{P1UQ-ExU=<1JHleyJDy z><8q#2cjKTU2fOQC3(*hdB+MwfL9-LDioLEHRV~XN64Z|?G%Dmb3U*)i$4F-dajE#z{rLIb zkN39BL+gehXO2HymshxZLmEYL?f1sWGx;x%+>Cl0O8~^x zq-fx?ebH3rqYIXizg4W*5-k_ntdQ6kosHg_;@!)5J!(W4BiXSOnse`3L^MQ#+vi{arEp=R z&RS1y**4xdNKD|hX%%e2V#YzEz1awm!zF#2V=Qvwh-zQukG@t`cs9_BCazJCKd{8Q zITFYq*V`uJ7f<%Fd*QHWcEC2cV{aq@-?HsMMd#gtRjs5@Yo6Qoy>QPBHjwXV#>Zfh zzd}@YFU1*oCrkaGSCBz65-;_5DVcgFOL+g291<@7fFm;gJq~^~%<_Ut8zbjY&&9=3 zy2Iy+TnZu!v%7m-0W(RQo^>zO-tdtSJLYE&To#QJe6XoHANWA3&+b4tU_XTj5|LsD_doaYJaJMxx*-|Z_ULtekwKQk8+l1Ngzf~t2 zMZuJXNtoUVPlo}dLYa6y#lDsN%sZ8Ie_MM5N zoZMA3w&|gmwd^hkq>!4lBoSTvbhjQhac{2ZHHpmgxl+v?s&<#Vlw`KdRz-x34`~x%GcpLs#kAo z3b^T++fXM4#!Ygm_sLWYu$qDAzg)7Sz_VLyyD3-HXHv=Vq~S|vcy`?}oMW=%OoT2a zR}8PMDrw2iupiNag_VxXdsgEkvQp4hb22Y`g%cids_glQED@yfQ%iyS6=xJ=&4+i*KutmJ&1gR zL=%$+WJ$P4vMR;+Ata%iR>jcyJ+^c&X{3JD%78ZC#f28d68J8{3yBpDXOp-(3!_#a zDS5{z*)uquzylHA8hm8gTg0N-kTq$VA7ArD!N`j>6@q16A!c-q_0E}=B~8}I3n{^alT?y>27yRg zPKC?@71J@tTRkM@C32G&a>H$uyv2!w3-D%Duq9ISgtJQM(EAB;) zu4o?n$P!a*CC{oA?=6k>N&Df~t-UJ+MZUvF>2<)FLDs=r5-P%^D@t{E zC}AM|f`tKWaX*lu-10EkSDL%f_&H9)vLN`| z&L-ANHPspBUW?Z=NbNZz6;FRZWEC4F@(da;_Hh?JcKi1w6zq*_8)TksSv1U#M#%Ck z75&zX9t(!Kvxb}6Qg-$O#<(7!Zp9t9iJYXUgy=uCDAA0TRIzf%9M_c3>UT8t@b8HE z6ZoH4O6UN|FtumN>~}`Y6AdU!S-6d{?S{hi*M`@^xAh(uEE%YN9EJU6M&EvW`Eu+w z>36Ik2>)(wRfNY6+&uhmm9K(9iGQ~_2G(J#9K1!Vgw-a~mt5V(wc*-%t$DTn_~I2Y zzeZ{<-FCtuLt#HySyzldu&2!`VYMqIw-lUG7j1*@9^wLUY1!;xAGd4;v+)pM(#1ObmUVYbbbLD&zC4Z)8^*-P6t(IGA4 zaqOizElN4fkgbI=|TUg9c zm@R4r13KHMB75Y)&EI)CN_08K=RKC{o0qO$*3PN{wd9X4Ud_q3MwKya8L66Hzs6~J z;!|#H)YUz}LJ*_Rhj~Ct*WdLy{1h7&wkHg0CWujc%T&C4&+^`I!e;%giB>n+*oa?ZF!GGf^bQ}c)|Bfp8cn&j z8&a>#N%Q48I-B4pU-$zHh3(xK@^35{UM%PjSgyS;TsbQ?9CLpf(qdRg>XjrPZ-WmN zBHb117c=|2@DGwfO_8nw$lW|!`*f2C<^`L%?lViCF{s{cd2PS%R@9~*YniilJbXLb zwnF(lS*#ZqU))BhG&M<}Xl;j1!;~86p41sGdR$dh?zc!btSJyP; zzt#V%u5-}~81BcrZ*HJ*FZhcOR&?}40w_VqY&HXv(#c=7%$#PY_#pXu4+?Fq?M^E6FmfK#D#_A$ zO4N3fadBHWFqSi28{ORI~&`R%KY1%X%C>hjiyv7{o z(JM6X9k4zN;W1YCu;KF!Kh}WdotAs}vBEIT%trgM!v3pI5Q={wr-2*&kK()Hf|s0@ zvKZwzXuRc2D3vPZ25%UcG5KKKig#elyIk%9tn)%a2ZBAn6k018D%{+Re?B9E-J3aa z`#$+sV^)N2AKCAoN4GDpTNT=luf|rCNFi7<)HFBlNd6X2_W0jK%TkI<{l68>aQogp zgqg3o{qeJFyk;j_AHzFmFR9c(v==P@}clRW6(d+;2XtP;Ms zg7dt=<9Oq8U_w)RU?(78b-YE(JnzoOpZ^v7jlAIsLUV@t(Zj=BnYOSXxm{_cljnb- zX<9r-9=tS?R56$^Zd)->;k?)<`Hi;zkTH)TYo<)-{ILYFcV8IUwHvK}c(^5ih4rv9VuxY^+UhL&6k|c>YU} z?!8OoFw+D79llzl=$l8wf>0lZzCDPd&;Qmp)tP2?y?^sJd0#_8=;AdmoFA%U(we|8 zj7dT--(COZ&BY&Hy@7qh=$3#*vRoUfe4n_-_*>g|zXp#@#5(On%UvhPyVutZPu-43 zuHS;v_S7A!I*DcP4m^^*)2lz-P*U}#Ky$ca{+PD8z=Fw7VuPv`6RY>E+A4FQzQYds&Pr($RdS6+vVE)=nbr*G4|C zyLSzDuhnm_?+%6~VyCo@ynmP{nBWR9& z!5A|1!POLpg3-#mOOH)O>}>X?B!&DdlnRX_BKB!C-eZ%hEYqhVh2I9t6jtaDx5pMu z8=7Mqc#_I{aZigh&@4JC%>fZ{s4QkP(V>0^g6qJitN+J>m!|f!sSxey+F@6TsGX9U zjA)W!zT(-%1d&NeGuzsdmG_K=rO78xo}lgV$!~xATLKNjQkL*d8wkOte*N_6oczCw zx1o@JV%p^@tXPFJf&=jZt26TN*WbLW>!g~$1_7H&^cqLpcFxI5FZ|B8_0e)Y*kK9E zB`=p=Xecw68oLw=79o0AXJkZA+op%eTxQ$M$aL4Tf#gUF1};N{ldtj3D+d*i z!4#0G2@lPrSiz|c_(!B4Y{XKR#k^a3peLdWwjYz)NJ!XllDpYq2QL zSvn`=g}#hSp3pL?nW26=Wof15YLg?nW<(Y#!}9n`l6U|O4~o2AzzMjX;ewY1Ew(7U z*t65b3i}g8?l8;>yEQ`gRao37qZ`@*o?VaVFWI6`_$r8ayO&FiX=(=zbC6-Tk)NP{ zget;-=SD*0_UgGfHCE2G23J-z%eFK2QIltW$6}a;9hlN;`XR9L43Ro7Sc!Fej&L19 zLPo|BJQ@ia8W&V0Rz2tg%1Scm{{Y(LI@J96&Plk<_faz0RB&I6j`?bn<0(YkX!Qh(0J z=+JpckIn-QY@a@zkJqX5v3hkrMz_vK?AQ4y9XlVTXXgQ3JNN9{c>u2O*Sm9{?ww=% zckb1}bFUtr`|QCUeLN5CN9*VL$Q?Z&qo-$lSI;pZP5;ja=K}w>bw>S_p+K1G#!~L)kdrAQB6HHHWISNiiT<*cWfiCyR_xT$M2!$+C|0A zNGio~Y0NvhOtJV=mB*|2UI@Qtrn`r6qE0pXisM-7Oz$Yq=pwUvK7=}@xa{LA4>@Ydwa- ziR;n8v-hBR+!^-~v|{qAb`?@ax80UtAVufj0#^Dccd9M#%Znr~q#+9D#E!(QyRmAcW zyVg2&Twf0p>QM>r8)Nnp`f5Q6zhb7iGPQO+FG+pPlURzyu`#cKXfeSZ_E~IEhk7Vh zRyj@=N(|`LGNrH~GuhDMmZeUzian((y?M?&QCOx2QFwIQD6koC$UUCVkCAV76y=P9 zG0L{kxWaE~uFxW)Wg)P{N+=FnuHzkYIA}y4Er0A|e;#Rj?z20ab(-5W#*(v_qP~gj z4cs~HV~9~{G#zqLE29iTJ@HeR0B1S^j5BJ51E^e()F_xy0ac?Qy5OsU0INML$vweX zUcl8Tpk8p+Hy(0l_H1~+-oV+y*ZzS?iEdwi=Ri!I| zx;N*<0SmJRM66_OLDt-K^s)_0jAz2;)$t5sti^qdwLMZD!m+0%G1j`irx|dth(zS- zK5?lD9AgVkM1qYeOM&ClnHue@w(W^4GuM#hBNr$OqG>cJKUUnv@^x{{f1KHC6yhIZ z4Y$Gd7gv`ftqIte5rO81W}squ*b7+L_vG%_{3Sj%c>w zA808}%0kV2>*k>GP9c*&420$p6*7Q7Q0%BZn$-Qh<$lJhiq@F;S>uqlyiEEWoNlu~9@W#|MA-8EZ#(Pa~r(>#=u$#L_P1psl^~3P_{P5>)JLtnx#=Y5>lHriuL> zp4}(fmi?kX+*Xbjlt+eh9PW$8ahOG>!xCeyE2B;+WQjSmV$WFRDrAYV)|GKEA0V}D z2WC{N+bEDjmKZpzQ4ohMF}`(W6zG5@COThk7iLuc+-Qqb*b+mMD7SJweqNRsPrR{Q zvtgAPZ|LwVyjjysU3)j8u0?woXO1(<$fD6;>-cy+KOS>Bu6Kb^+p2Oo`xnAQB2$|K zwB*IeNopazwgLDSEg+N5)-(Rb;QGcaa4*(sGqPVFEHKw%va9+ZQDv(=TuTxoFV`94 z8Q*Y|4W`K{QzQKhCkSAk(M44?rv1IP{ zFFu5%9?b)Foej}K=47zZ^5fEVqE%@{r%)W*{lDG1af-%`~Nj^q7A)(i*hE586N3#h=3dg&TjW%_qt~N zb$ga^3IYKoL6iGWbaD4n{|3AA_s#Do(f-D}TRs;T$_U_C_GP4v8DHu#FPlH6w2Vo4 z{Ol~tKSit>mT-@RIt8kI!GUUc(hbu9#Gj3sYebY9#-Bw3jg3+R7k4!3Pf=>aV~SFn zqSU4+wJ|WKD77g{ZHiKxqSRu>6s0yrsXaiH8Y*oav1*ZvGyS>hK(&9QI5njaEMXT( zA}dke$XgZ>fYP!h>_C7PQ*GNqj4c7I54+$VH-wE=Dc5TJ9yn%S^_x42l%?(uktk-U zNvT&mCl{n(3D3DHlR)uz#5z;>IF{b>Qn74_QV|t(+2q1gij8u3z?4xO0*Z3a6(edb zt1Kl2`>XLs#}a&8|BW}BM$`~&v0K8piPD^#-ZPIur`X4w7l7>7ERiDJ$8fa+qZXOK z9Tsh3Nvp!8s^8I!r@Y*{h%mffTK5>N%w7+Rn5sT=Li4szSIUc32tr*rd z9#uqanY%9}%|7zMVNre%(BCS#*p_jvI?S`p(tQ8d{J7ITuy@{GTS3!4uJ*(YgeC)A$n-|je6gTey$fFK%c00wGC@VqciQZt3y%|d zFR)b`BP)K#L<=iPgvsa+G`VFWCEqH$Iq6o=jcI(F7xIp4wl;I8&WlBQ{{Gd)%eSxQoAi(-lOvmG9fv#EHJUqzd7a*h zt*ppB#X5_o^W6>4pOaTYRnTIGmaWeW%4tAK;`-o`S7vj=66Gi8hIeRd*jl&->M%NI`;Er?Merl)eSa)ocqjeOW8#Ki2=Ug16FCK%!*dUPb%MO90>4p#F zM}HP1a#$q;xvKZCmZ|Qmk*S1c?DCr^t;{^zRLri;k6wS~0X3wrNBEx|@(>}@en-L7 zAH1m%d03Baiw|YmD_XL98qc%M^LG1Sbu>*2rj(_i21|lVytuj~e?(A+_bD%AiQ?x= zxuHB5sL<}A+j!9~%4e`Ki@9=ehyk>eg;(OY(t4}WCx61$&^sd0m(v=0yvS5bOrBlN zrWj!4jy|NA;{~SUA{mHcLUC{cda*2~g%4MCegKM9Vax0rx#G4Fm_{GmSjvO-XeT-7 zi5?EWW0-icx5Czuf1Wo~IX*6}yvUIN;Z%e=}$L1^AKAMAg?Ote#QCC>N0 z8{0M~wr$%^CKKDXlZl<|*tYG7ZQJIKaq9c|os0iX*R%Rr)zueoy|=^%;SbKX2X2l+ zEMB+FM~)4oEcD{b#c`(^ zZpO3XKtA%Pa7M}NO$khJ*v$DxuP&8q#?0efApWs*bGLqNfxDOzK%23|PZ|&Ri>*h_DikA$q?isZKB*&HEJ!LH7@$O zY&#cr=GEle`4)dEE{%J|0YGNed@_N2MhEKYbA>DmU89#b#&7L`V8Y=J(Wdnr z>~j#|u!UPlKy7I#@@$S+pi={f$zi!)3srW`=InRl?MIp8J1`cG9Y!C+HEJk{LMfhW zDUgU+j)zGpxSyE{B?0m!D^uPtUPP)FgoqG!tY>&1TysvCt`U?VClfc77sp<=HR0&x zV3Sen^F3r=8OP?bhGJx@Si&(ioj?17#OvVzkTV zb+OmEsKPlAH=8r#s4RhS$oViK?Ad~Wi+49*B>DPy@}33!HTPI>Vi^8xcbvE!k0ivQ zBA_itzmli3*U<~;Mw$Be8|_KW*10zbhG|>zR07yeJ;Aw@Wz+{PviGu z%Th1KG>}<>XVh5Bj|!s;w7ppme$udlbD^!d+*5HSH#$xEc6D))aMqneq|*tq51hjY zF9TIGVmB-J)it8O1Gq^)Vdx$kX;)mt&_R)J{zIEC8Wwf^k0xs*!)H3Fa_<-`avvt0 zXe?;$LSraHN0?1a(}`v%w_>GotWC5*m2E*na@Y?a)z5~B6odyLRX z0iUMJBxB_5P4auJDQ3p3f!D(L`I-Zdx7Q-t1@Zn<$+K_9XZD_S?z*T0O&3Hm{u)QC zL!#veu%1=2KZTGV(W!X^re8^4GZ_X`0b$5U?6BCSh4D#%uECtI{`gf?FD*!Zd(`rK zErj6OW||U-M;)ZzCv*NEq0}q;JV^>`kP+W|?Raoqc;g8Y-{#05(T}nrE6Wjg_lgwc zrGG-vPyFR2FVLc{>3@)sE}%{9N_oPhAN0od3yix6^lmhZYt`l9D--sMDFw$4phc+YZ{82xultcSJ}He#-NjaG zSX>k9J}{mcrNONomLzJPp$S>ZUt(tGN)O+w+dyWjjpIIzKrkO}{Y-&E>MgBbXWCal z{8qYwqGE?tJqNGlt7b-lDX*!^;T-6L*su;@(~&m z_Z0=1v1wOG+Eu#}H)Q)+?~kqy1e9|(o-H(gtjxz-4Xh@?lU9<-iwY8x%_Dp}{{n~0 znwGI8w*Ctd?0kf^zyQAj(a(~Y6JXoev}izXEFk2Qi4H2b|CWilipexo5&Ugd-+HEN zCQ2<1ijyy|OW7p`fnda@nZJT8lp_@K*SmFm@*4N8$aN>0(oq*dYTR&3B{#^p{llI% z*?K56U8pPRMvAw%W7DQL2^W)YF7I)3*-O>-Q8hO`gZIbGzblhMwLB?`b@zk#$D=Ej{#xT&J&n4Ye9I_s2zA#V(E91;|O+u|=G$FUT@RnxCvMIDVY0FPNC~ z%`$#$h%gVhg zKlrk~m_MYE5a{B(c8N}c8E(+KVW40*C?rhup}H{ z0$`D38TP|D2=`}?z?Om8!DDK)Fs+=-4O-PgU+#2P1uSlNclf_ACk*lbrNn2HWCYnB zMs5$ZfMx*EL6=2l8qtF=lvcrFwb8H?*MXqPkGzCj@h8xok*r*hFSEqOnw1V2B(7Q+ zBGpTThLUT-!mM&>C+N?V$U}ugl!GrF9QX-G2bWqsT3FH#L}Kc70?OyvtvT}e#tvzD zLTPs73h|h4zPxm%Lw;G;wYXML>AF`CYsG+rNHf0$-B?@= z=dS1?C|BIRsM?HS-76DN;hZ*AE^Wd6NZEFd%~qcJzQgu6_PX6nVP7*H5? z#h|MLMu7h4G9tEjU2CbBqZTb_CY5a*hJ*M=8U5^$X3;%kW4MWtXb__}ZMT%zT4HHZ zzBm2eu&Go5Aar@Tzu6yn;Veh(EP3-U^BAu129Is)A#e4jsyPu2Pua~8#h+^H{8fO~iuhYHt1%xD~@rc?w z-!m=x)M0?FT&^G=c{H5Rhfy33d!5z4fbUgb4~O9VpBEh)SbbETpt1!B8BrH4g_cPy zu$N%FIk2uL(kCQf4H2WN26H7@!=3>Dg$K5$MvssI6oR3G|9*=| zwNk*q(>5bcXZcIE1b2aBdak=*xX;=f)z=&2Nw=Y_vE0{0JFN(Gm)T|<`PJJyBh2Ss zcxxB`srqCS_|&JLLRz)E@Nk-P%tOn%V5+oEqR6AKX}W>@3IMTRHoPpWNz{-xZ6=Z? zJh>r>XRp?`+^5=tt;_7jSm;e&nuSiMVINF2Pej^APLHta62pE>ro+6O1lOQmJ`s@t zT`d`LiB>0DusGB_NOH{LSw+uaA!~yO{w6&P>_x4{Dv@mdJSo||wAv}7x?+UKRu~bn z7j-JTyKsEtJF17b^F%(fk`rHdsZ{W*wPT?B3yj z@H`+!e|z+Y#56N@mR4$P^)}nOv#C5kNqIb$dW5bx#Ate{rNwTX9&hFi{wf>cbj8WYn#V$&qaSO`*m z^^)oe3*{(iAzla_a&sM-4aTj;#^sJ?>FP0=UOiELajC#t8DiC(Oq}& z=;)$&mlyi)Ozg2si#nK%6M4xu5h%GioDA!58z}ofxruSp;eC52&BgEx$zL772(B&a zYA;Ppb{j>4jA30Y<<#zbl}~i+D%zs3pA{yU6%@bU*r`ifizhXfIBcS7?9>Zam2*>_ zo}^9Iq6atM1}v{?{RQLq>)F=tm|eff zTh87~IjRBVA}i86oueE-R_K|%p4Io-%a5)*{(ym7rSAY2>YrDKAIZ4gCh@3H3wi;; zuR5M--ISuq|m$eq83m+!xp(s^d zfgqrZwE+b+KbnrGf`Yq8KB@5mf9+@9L$Dlmjzlge<*AHU9ILWqS$QM1T{cwRhiDXd{Srr!c>JtCc*-_ zh&j@`Utq1_RW_d8gQ03B_Mk@M!=m2<4MLx*1_>^*WkaI}IpExY3h51S^HU@DiU^MQ zxZ#nzB%i~Ff;3QAEE2T`p+vW0?4a&JDQ%6-*R}Pm5C%1dv%a4lhu=FwVTp-a%GQT_ z2)5UL4t{QK_Z$PCe&G%R9TckPv<-UG5u=PLik_SN{pDhP7)x`Gc=-TUa>?`N+K8IFK;2n!lG*3c zkI)U~&y#s74{%QDD&bd15H9nWU9x^IeTl_10qS3m?j!WJ~84SW}NFJ6~A;h6tDceQvjb&nO@uCGQLQJeS;Mr`kC?@u?T;yA(pM?VAN8orTn z4u$HO_5t4lCT|giz@hz~FaMmI^3Qy5GkFaEXoDV$KZ@J&1{{X|mhODOGQl9~Ey+?h_(^6N4Z3N@d@AepKRdSK z96?T+5P(}6CyfnLBXy%01v9yW6`r?ZDy6UbbUPdU8!VT*4q<{*R3-scvuccE32iA% zM07jknLY`ru-d^4D{o-!DF!Kg~+W*<4y>s6I*olP1D{X*>C!{Vk^^M{*UW z9N(F*zUE|@0gJ}3G6^dh(@P{%eYX_^yeK&Y3RJF>eYI2XkUd_4isF2ptTKIy%x`-O z$jM&{7Jr7ji`G>PKbOLzx#X)S-0TeO;|E}PmgUFcNA$_#*GU%DbUJj#W_gW31qIf+ed@!Mu*2w^RRR&o zTFAD^3V6cg9R~2Z8b*@L>oeHbYx}PuEhPQqS|bjT>l&)hF8!wfY@Dae9k<*qJRqFk z>?M~i>qS1&{P15Q`I7!vu1@h)gI-HQMKyFW-i;DVVZfW+s(T>GOwbRQO74$|6?#|K z^02s6IcgW$S+S(N^}kugSZz@lMQH1nmF~Wz3SM`nU>DAsd`vMFGiqq)PQGxaj0Bpl zvmDjY=BFQRKTgdSE88x2;$D5&bANweB83KZQwv=Wj0g)k z_F+U|5YEz*DI~)?o zwBrO@jg}P4n<~SXpPNQ1ypy7AR|Wcjddw(K(D~ggCub-UqF1uq5c|C)a9HJ5F$0s= z#B))}es0rc*@j-$XjBbRF^%ete0ax*ud?cW0RAU-_H#-lh?CKcNW0vYh%Wi<9Jg7b zS~i?UPZ#E}#12HS7G9^u(NMn!3HRbhotVE>SY8B{e^7znIn1sqRwdM_jf9F(djm#d z94sswW082$^sDms`H$0{;!*|Q49FOVVa+wqV(>C9ERxGRT>^sUw>7QwV7ahc4;C`I ztfnoQdmsr}#*(5oF&CmcVq6l{5kx{t`Y0UZy8H>DSNwN7NujeVIWr4`)w;uwRIu`Z zf4=Dwfia!W3WY^@B@5>Z5zADG(S<~go7)y4h^+2egr1=KDqF8D&AQ0hK9_r1W=8Pg zG{r|TB^V{OF&>!dD3dnlnpE_^t-m`mFN#=m)No|?GuZEb8yj0VLvG>Ttl>V0>Lb+= zY#yLv!K7KVv1hd+888jho+iwoQtG67eD2Af19@OAbv5Z$RHG)I!dkM*T?d*nBg?d9ZaZQD^t*wgoM6)uv_ zho&FTEm{#2Z$zw|SrI7K))E=0Ow;`Tjv1D5;ba8%abeS>pm5@Nonx5(56(x@kzkGq zz_Y*x?&b;`%9^dY&X7E3JDu)b1161qKf8tgBP1rBWbRC^pciXs4a;)y6%J!&C^^)S zd9DKME6$DUSSO*&&d}N7DK7Weq^V_{3nM!Lb}t{b{!Ns2U0~$;W_{9JqmlgyzPma6 zAt}#rcRz^y<*{re#zj%twMz{FxX~DGM zMw?q8TQh{Z_>5OwoTl|(v2=q;;2p`I1+Lp6L`HI4Z9V*|M)_MoUORUx+~XS3dczf~ z;NKHaScOU`eoEN4zb|sm6{juj0;kLpIMqfF1T9DiyY}Q!L zpY~bKpMqGJETyy*%Y~)!J@RJcRaFP))I`Uvfy9PXp_Um$Z0WrXhV%k+>a`gN`A22&2hO0h5B^pIHlJogBL4p7 z+&hB_c@XE7W!FkWqGyHFQ1&wol2vt3OmqC$n$gqoE=zi>zOS7=eCE`1J6P99Vfec3 zuYYlGK)?))b!J312gCEizS*+wBd+R^W4s>XQ4sH^zZLpv4IW`U@fMw!HWcN*3m zPKjr29vD#v5>aaRegE66`z@z~{rzmc?@Y~? zTP*PmMRKk(th=--*ax>G8Oc#u(4&OGE?J_g^qi=t)!U*0fc^ILZf)|QJhn^Kvvsml z$rskSns%R5$MZteovn|IKDVnCg5{@D1b}Tn4&fWFe+!qB5et`M9YF-<`IooR%}9Zm z3+jLt7-e?h!zW#=U332xpfa9h?TB5^EA)mGp+k}0XXKLS{ey=h7r29w)MLn|Y&+0q_!-tN`SZ02w@ zR%B$QIJC5f*F3;`J?rxBGcSbYwa=9$4jW|c-h!Re=`GD=SW=#8AF>E$>@VNUzAd?v z=c!%Tk4giVu$$sP3~Noa^~(kd;l>y4;ikGnKb+VEO%=XJtdJj{)??2M9m@s0N8bbt`3ba}Kwr*t9XzPq<3K zSYm7)wI{UaqOPZ6cX(oqZ^Q8F9c$WcsYwn{RawX@_p z%l|K|oYks`<^V;)?;DQf z&uizCxjTeq_-)r1kI>3-IZ-{}0@YjrEGGpEOYUzj+5QV~k(@NR)(lnj%Sn-w2h!t_ z9Hrr4sh38)Xq+H_lV%B0c;&g%tq77hDnmJf0}QYtU8F{nY3DGkqV+P zQUkv&Q~}+Q()pQ5MX9S>bW1OPuJfOkPci!-(_nJ&miXkGwR?jQ5{dn^EW$hzzf$SJ{fs zcz|{BRyTT$qM(pmUJmS&^{xgWg&eeh(2Oh`*!f1xdn0})qEA~k)!ZBCU10wb z=-R@c_VAdJEnF*JYq&#q!^7f@ktU$(y^5j~Su-%6)G2O9fw&Tngkjk?6v%z)B3Xg~ z1?3!%gfZhzb(CU;;m5a;B~EPv;_&FjC)c$o)vi!6guE&#4=& z3;2ciAXS#@<^#FMc!1|}kdOqbc}~+L{l#DUa(SH~bKo7xL_4V%`;x4Nn?n-3%5auX zgCE;cGoF#c@h3}>?q~T$lp9He=Tni- z@K;RTvHcfHssmQ~rtfD6WH6BN^}|B4-0wx9h4FA=(1;c&N98dE)PguU3`f^DBy)iF z#~Hwb$e16?>44{hNV&lM$$<8QQCK_O3cui^1&SaYAB>5Mo2d zMy%NtpqA5VG4ka08S4g|_{kx_s?R1Hf9g06`(?&nZD!hwr`LWLc% z)NkL2_P#N<=gy049^O=5vJaK)*L;$$o##QXY&QXm+~G5n+@Sf0@P`0OZcsqxt-OLG zROa4O&c+gK*-vq)BDWeASyoNKHk9B_9FDrtVI*RyZ>qXPAi||SQ-Zh1n}gneuU54xS>Y|GU_Mp!#3yMB(N&v04A$ zI`97f9{ul&73hk48F@xj*^;$N4t&{jWCD^rg@lSAg}`BimZ9?;Z;)rj_&ihrC$K;G z+s=}}f3P4mUB@vOk?31Gn+z7VO4%TD=H64~x!j=TwR$s4=oS}|Y`}lGIBH7S75xUc zg{5E~YC&E9Ljc^CM0`cR!4?2+3z1(Cvwaiy|I81-`@pvsk+9Sr$xxpwqJnB>37v|$ zG?&}-B9cvI{=eDcB2q2>KN^i2H$#DTcE^{0v3gk zu$ND|MA4KUGOe(!dgp0cGl$P4Y=oc}6^YJ;sa~mV_@wT?uqD4k+egfY?BAzpBvGnJ z_kwf^#0`J{$Q?b%8p*|i==2{)VB=Zf7V!MUVbTJAEo@*vcK0mBjvMM5kX4k=GI+Al z4ab1X7n^IiIO`;Z!YORT-{cZm$@;mz(2q?P)^gQMw9voCvuC3%>R!)>{otvbjHNeP z?jSvgt$hLu_hC4bkUb4#dyV#hy2``oUmeI6M?=B$!I)eRX_2e^Cd9%0W+H;ry`3NL zL!Z=lf2u3!X|yNA$It6Wme%NwJV!Vo+Jg0)ivJwPA@a>=-yXS-aaU16cGbnJ{oMYV zAm$yB#jzsZvg$O!iv5R?X28dg9=0_F>!l}A`pAe=kQw>7G}vjT0u+>_DR}3XWCFhC zjOaEgTLOo~d8iyHiXZ8ZT^boD^#z24>XuV@(oZYom%PX%1|L*&CiyhCoZJKanWFU= zHQ5;arxG|TU4&B}4F>}XpD$0c(s%i(u&u0ht2Ck*s-|2SGJQ;MVz!(UM`)5Ul>!1j z=X@Alni*quUThMG+jNaXDK4(LR^{L`N}^h)wc#@o$&0kw6Ie|`%)ehz&Bj$LBJ>0dpl>rn-)kdCB zoRv`-TD#vZCd)2JG}6N{8aL+dFM^zu+`skB-31nXJ{8A{<$K!3@n%Mq3_or%ZPR6b zGMzO(9WcBfs|0-hE9AbxO1`Fozr{!U_u=QV4JPL`DRH;M9(nKcvhfjaXC4hIMGZ)0 zhoh~7+!(PE!*DWfu`@Ak=`k^Fna2E1Z`@EtLv=xKwaoUht7D%5CA89?#j)CwJ`^=6 zb`le6pzs)hAAPq{fPxsP2n$YudgMN6a7eI*>-)mFl^KzQrGWsw9fm*jcNtg>KJvBO za>ikh)qv6`4K{_4W0hcAOzcj{1UM+vXyL?n>Y=R3^=ejn68W+3cG>*9zC63Uyj*{* zUuJcM7Zq<|fzM03(avs*bwqA0*Gzz#o>l+61RRB&s{d(OSQEJ|q%tF_1A7N{fC46_ z7-48-JKz#pzj1oo$6e<2D&d=^HzoRcFm*nx(I(9^^@hVL4SZ@&0%%b?=-RJ5COF3E z%&Z58|H&UisuqsglOt&4;&ee(x6DcG&HK3ic_upbwK)niBefuE|pp`q_Lscjfjt zJD&R+toD)?eI@Yq4Tqv5H+E(WI?^`bL7eu)4l}r-?g+K7 zVna`T7xGfJYxw2t?e6aH2At$N75a3(f9Ux(#cih%a|emNkhH8Q>TD2@bQ>b>V{-$y z93smX%QuLG-f_ZEBDaQfodt7wD;($nPEihAwOI0_o8sQh-O`^Cb|0uYfkO{wR$yxo~}7L%5Zcc z)gM+pHhG+`5^3GUHwljzaW&dH#L)Ygt{1R0UKQlc>0&YY#DXd~|HGC82e5`qKx%ldS@So|) zEBW#9dK#XmLe`kqpqv$+j@zAH)f($QpobQls+u=)_e62wUt|_Jr(#S~u|r6Y5Ys~! z!Op^oB8k*S4@$z@`;Fcht8j?O2&IVQzR0@gZGR1&hjMY#^tvdfIZ4+38;ZL*EIR?nUJI? zl*Z6`RjpY=sXRBCKWlA=FO1r|rfV)a!Z zX+{MWCeBa_mPAxQ8d2Ih5jP`0!p@IGGL7SXu!sgMMJ6dFSbuvC%=jbxZQ4*3R?zP9 z>3$hJAE2_}#}BoWFfADOPyPE{TInd%p0IJ6Xfy1)) zu8(!8zr&j;qEz4C=0gG}kCk9FN=~Ch?k4X*@=DV3k;?JZ=VwjJ*BtB|n=k-Dv92yj z>*XKSG^-9@z%9dl>iuS$4H>IGMhZjUl&ACJAZTkPZcDG3T3WH7g9LhZLFg$YXGz?K z&5D*%X6I)q8KW5yx`Wb}uBm6FW_#=dOWc?-C_xzf?(dJ;ACcLzyf44^rPg03z|Z+D z`urZvW*w{)+vdb$Ct^8V$B+3`=90Pr?_%42A^tg5wjZmNfD{w)sMbtZz z-)<9e*iIB&ztv&yn@+SiHe$(;+H74;rW z#Z2{4l{UF@QS?io;&jRd0XnZ`o~x%;6^(r6{%KE`m(3TmW3Uw+yG1+{lUK*YiA>hr z*c7Z|*a(PVDe9PXmsF9QhHx#+)9DDTTkABOR=8g<7dyZ%Nh17~;>6JnmWQclv4 z2CHIn_ZZjW`_!Jr!~_H_IRO^ct-QX`;0#d)HxoIv>^#E2YeMIwlV1ZX7)un+Az!0o zMw5hg3?s!!hnzC57rSc@IsnoxGcA zf9sW&^=RCCOMlh&^HjHAIWBW@{$yQT1hxh(s-w^-*jOx5!rNyi zTT*_HCTCC0Pm??1#pHTeOA5j}B#1G#VC19ReDs*&N~;t=e+l(hA_^#o=r zh}RAi=M8cTl78frts|0H|T*ax%KTML)a)HD56V7z7J=s zZ0D9_%iYK$9X6(X&2}Xw!m4+1$t=Yq^WYLB1zrV$$^ApL0d=P5QcWbdmw=c4niElB z$q&h#m7nguU&%)PMb`SEL|DDVxtfMVMrpa1AfK)liWFWF7g+VRSFZ3mOg)OLK+R+} z{K6vk-p7a?;ebC$?`k?g+r@K`XL3 zJVFpX-)cAHRg8_y5djt5ELiDjW@)%V5KeymLeeB=gN2-eVu^S=v?$ZKoK=)29HoGshoZUwaN#Wn4xfUx^m*V2a&ST=lg}{>W0Tz%C90^~Y zgejhD&}BRH+~Ol;g;qg|CO2D_DTs9h`OfBTc=+h^r4;gvF2Wk8Mz+y@kvH$tG8i)% zSxh5S?^$Q@p8oM|ZBg8VwGY(X>)IjkH|pjs=2!^ISTu)Xd)TA~ds`A)$sX)MNbY(y`n z^qZQibdCD`pj!8wtn>#mxY*2iyl^O0iQtk0qZ7SW^X9L-D@NGc$sNb zuJ6Iuxy+9(`~WazN`k#=vYZUe;w%!`ENTYWCUZfkPo>=66jL2J*R>yajRV{Gk8Ha6 z24uCT6$Ju0YD=Xe5S_WU^mn;~PH`s2sICP@8rQ==#{ror%v?_o;6Lf?U`>^rTvi!w!75zL^fgM~?!8mAHX4>=~mND6=6D{$6 zBI0t7KlSIE103?`1w=ri{e$9_VJK|lY->V((oPK8$O*$nI~Bw|v%T{}w0q3!OsWI* z;gPiH)9>c*>+bb*bNaStvPZ9{N4n2KDA)`-UQ+Kt5udza-Qli)nUuWRxE+@V`y&=?0WSwD9D^k!FoNbzfru)~-~$WdHZ__C z0O@=^l2M{$#`C=Y4V#PX)P1k|s|C3nF$>wW-4en!AtRt_hGCEl@xc#mAz#{gHH*Ux(kBass zzM3CuT-Fan`V5P=;TP+WOVT(oGSc_`^c(UIKlf_~Tf{ms@I&}G_NxOFf`3kJM6GaK z3_;#=U_zyA(W9%woM2Mtt2e#N}v@a`0nZ*ab;D- zP{p|xUnAOCdajjdLrVJwrLy_2;ILsVr&W&Vq-PzGd=Wu$zHQvLkW2$vhbAPYs#8$Lv?MBwu0Z%#x$qL_$%u72-$le`lwYJYMz9%$x9n5iObYic zF0GvPEV6Qky_7v3rSIn8&n6uX9C*5AVW-28H##(By4+`7t+6C$(C>D1O5)1p>-(QA z46Wn$FGu9KYK?p&quEIYhvI*e-h^LrqQ1_|@Zm(TnyFS`t|7*~HArbp7wM@nzLxUg z2K%F=9no03tcA14_t^%U3DY0ptX@bM$?&#Vg&M-n<}u?W7 zFAOV8{8Rj-mrj0{G$Y8mLK|qRuEEX4$L%3Kqo85!9k;{Kzo61}^O+8AM`p`R)!{tE zRp5;!n_diN7gJBs35WBvi`ro&;VN5a*cmf%m01xdD2g7Y&|)J8x+Pl(wU2-Zy9WKX zL=>kjzC$Z^ITZ{RG!ooXY!|k$udu~*Adb-w0F%z7%?#N6fMM+<8K*pFjGjNLjvS{@ zbt8%$l1s0#Z8Goy!d)C|)7}(Va^M$4PlAT`h-8UalOEyV|w zON}nbYK9re-r?G04gPBy-ZDTkDdY)9WqDVqnkbXZ%TK_LXp$v~fF`J_SnHLcxH)^a)!o2(hh2HWt&KC$Hd#@URN&9vn+=6iC@3)YiF7n zD?NuY2o#{;atou^F~0ic1Ri;EEwX6qG|<9w&t?T+M~^lkX*7Hl9N{vCS|`NWiNp(I zUt;$|J5r)fEzQc-5BtO5boB9APIyp0z@vF6Fa_s7Nqm%$AN%ZXbp9G9lNDIztZ~(k( zJ%pCDu<@or&Ql{lI~nzinpP$F1q^`WpAin8qLPY!W4zFN8*ZS$j_qMiDD{=PjUb-T zqe$*BDL=fZTOk;$o(UG{Z+qo7J9@w;m_f!xqyryZ%~^$*fNjxww5SxYZ2x0a zbAdYblu-On5XtPt~Ho_1YBb2Pkf!p^zNs6C7b$k;-xklgols-^PSO3!OrrJbDajX)2&(6wpT zDxfM)_9TH3LrDHCfnI^(q@Bn|cE<(eSr+r3y~=pXkEyPk3g!eLYxpta1^%c&>jCPp zs1CZWRf^PrSW|(nIt-eQKo>;F&d+^Tz;>kcM#UKnCKN##s)>V;&le2hZ<5e5c`dsG z<0nh7#Y_N3PZ^AsA8evYP;5lWXJDqiU=DfW8@v>8qVcFEcmKbL@?2DTaqX=;>V!S; zJq6;ZwYgL&;reC`;iuY8v`jo`riX=q=cw*yrW$`Su@uKHfcRwB#|=6{%TfTWC(!1|`{w68cw)!T%*=^!S2yvEkdzYir; zh+rQi@Di7A9u01w4(I5SJS>=R?MoV+2#R-CnFr&mq$$bLz~j$_K|iGSYny*gA~0hR zqozIGIW(lHPh@KDQEZq5t=Q3W9%Na*7cGd;eK2W$3rv1hbfOJbJ_&yC6lMkApKyxFU67KcLAX7wq#LB+VO@N0J$LxrgV;FH$^L z-sV}!Fk$q%hJi2y%0P7a5T4;6>3^0J;U!nFYR$}2f&gEZ6ThP%UV3)3%!}*aiyJ8q zkGKu#Fc*VDP2yDFTB`c%NhwDou>+(el0|Rf`>=^91GB9_G>B23=@+y8_|pUZv|#2H z;oXolgc6oPDwjV&0D*@@HW~b;5i^Dqsa>x;iIT1)Nh)1*7?M?Xo4!o{M&4!v5;ub> zjQWb6nzZk$&|z3)dQ1;veEw(@6JHqi8BA3tGH{Vp);e={NGB3GLkgy+u!gdQ!$~ld z@~R#HY+Uo#z<-v?lwy!JWp`W+7DpxZWAGJnG#2q33ROV@UE(iezPzk!qVmacE;NK4 zvrm4tEZv;=lO?&v>=ZU?tB`FM*Up zj?qGK(#B1uKnWrLzTtbg9FHdNx76Oh%%b6yzElR>?79p6z>mn<;-IbEz_9h?s;=Mv zLu60i#h@4Oef_qWsTokQlrcD1fDEYom8xrHl^0f8c-1FKuytnJh&U02i>mT9+zySy zgR97y!j8>L*%XchCps&sVgvio^7m@W^{3;MaD`tO;^$>-7~*49WJy|x{zC;@n{Gmr zsA%#7X2_FS9B;J5(%kh-2Wk}}%t>Ppn<7)c$C`V7@^==;tDXbTBtPD;sAyq7Eg}H6 z;2Q#iEFgs&U*^i&9r(lhdR^Z3*sp=&bzN*&X|UP)4=d0b6T8%qqi6kJ?PdA*#;fFF zI=iwhqq67yz}>P+rJm>STo<;idmde5zTPS+G9EWF!AO9{03pbdkV8&skABSWyu`ef ze#ZjAV=7y%6_GdJ_yvC5hq#rd>NFCb1&iQ`;GdA1f=Z9TJi#Ye6JbPbCi$8`Qr5?X z$bk8FJOF;7MI!6L`nXU^wRuP1CWy;}w@bbiG5R+@G&E~({sy+vXV9u3g?yc0@S?Kt z%t!632_t@o8?wJtyq71_Q3q<3l&QlQ%TPlZ_*>615WK1@B{=662#Y zQc!u&4HQz@S2&MRI$>yz{(mZFeUpYOdM(CcaBeT6Jz3ER{wyQYQQ)r7ef&fEA-iC@ zq7kLB7}5E1R0z43^hnxXh{PAov3#C@p#>rm5xTo+u%LakzE!i|z@lIU3X-O9p}_+4 z-$f?^SV`qH4zUcaB3PtNZMi+jDr$;|2;bTsB8SR+h-&c#wC4-1@s6(E<|a-byV5j#0r`+{g;e; zBdhpgR})xNdnZ5-IQlrHJShF5nQ~-&Y+8l?kA?Z@iR7L%t%~o8-9soHO&$rOl+5xp z74s=d5T$SWwm+f}N`vFY8;<<#znho~mZn2<1%Luu@<80UWqwr#@fyW&qq21*NrV_W zDg`*9DH{2;iM?&dSDXL#1=^jW<)wB%1hO{x9emna=LbItv132Q@hs)zW{%@exm$WY zmB-6(GS89VG>sGPN9s*0(aj)Ascop{&~w>mwz!K!9j+kYM&GV~OvH=|QaMcb*up-v z2-e@>pCFwtYkdDc3`Xj*ljB0;UNU^k(@pLEJVr#H?x?UK28UJ$(bOc<8L_8aC*uCCHnA1 z*ooN_RlI2ZH*d_4Z#J@IZa2+quLQletx^dip#CUbvIc{qUKG`GOFgMCv@plPA4^2+ zIZ<#k1Z>Fyj8m3(Vaq@uK_ZRHSO}WZ6r8!V#tpzh_6AqPp9nem%1*7@8fNXOl;hX5 z49d8$<~H?R&F`{L6+dO2k#(p{zSd0@&8ric3- z2&71`-=^X{VA@Q8u&d)7r{3#nFe@D2I0B=&$dlrg_&gK_5dk*l8|%1ZZkt{dYOk+u zx7E*Zm~!me*>@kV{r<28^b9a)UGp5m6G++9-SB?pUJ$3m&#;Jg48Qa0iQumEe~I?D zZ2eu^UmT(y_=>hi85Y^pdA{9D85UpJ|H58qxS@cg^dsWFNr$F0SV4qKx2!BUpLP`J zacQ2&1tMDL=#P5I8ln11bq>DhWOgN50nrQb zagb#cMJcv&gopOO$mq$*$6}Ncn~F?82oL}D0K7>(bT=>jo^H)fPQk23lClFqp-~V6Ef<#A$v^R7>Rf2)&&A@~rEM#aQJV>qk4+I3LUFj| zAe*O#*T+o$x0;n6!}sRiYW89cys&iWj&>I_meMr#y!N-RhgGajq~qnE7=_ceWmiA^ z+^OSTC>d$b4uSO{3`JdX;yLwq-&iYdrWatJ?hIO3)w57^_l8yYrGc-Q!7YPq96To)K7U)%}srq+{#r_=b_1;?g>mp_R zh5_BswINRE7w_O*k?!gm_0J^6(xJbOEeRW3?U=j&Z>&)PP}yzVUEX$<^#UU8^UJ% zpxE?qSvEDSMi8bed);wu@WiCp2w+(R_Q$8eIvByam%uViBoXwvnHT)`9v&F1ToD*3 z_~UVJAD-mUhsoRE2ZT2c-d8UyDkRi@4Nq>o)LdG=nI7D?;g#5pxEBccZwcA=5drp& z-M6GR8V|3QR5&!Nw@gk9-;YqPFMd-(Ij`6f{xE^igEqsF8i=%*t+zFDU09aoCZsdp zsU2EaCS5FL>YN?nv@sJb5HpjJAtwi zm0-3)4A9by(3kAU0iJo?s0)^b;f9Y}ep{gCx)eQ5$FJE@6nV|#^o2T~hUzK~sjg~V z(mX~!Vq)yyab9Xnwv^oj&r_07P%co@CrXXNf1dCWzHx#{T<8y~nFTIq#t?QjI+C%k zk@xF*-}!@*_Nan0n`V$R=c1%+A1kcIjwK;DUkwR3=w9}#<5n)d7?mUE%8>L%C?oh; z-c~+H5A}7}kC8$=8hh)LQ!lLY!#QjLV4226L;y3S&EPbn{<4iNhaJWUf#joU(u#`a zr`+)p*YQMeQoL(sD{YxCA;R5PeAMoqM9yb`xuL8W+(Q6RkOUZGA*%8!;Wh_bp}Xc= zF%}(L1S>q@2JDE_(A3xg-nk&*gVM$(sH9S_G{&H(EvjIbWR83YltwEnrpH=lq=}BD zLUK^RBJD*d?GV7^L=VJfsan4@Yijm=tK~(dCd@Pr3EM!4yzc6qYp~EKK)e?nkRdwF zd9Tb$r+~Yim(3OVeCG!CpW(l|YjkBCh_txn!2x{#`6lIS`@9(Wv&CZvF318M38OzFZe_TGX zY!l{~Yet~#i_^WSw_aocx$K-LK^lhZ*jt8(Z56Mhroq2OFCN| zOL`Z$t|A2|b1LC#-ma75z`PYOa$;%rfI)$Rimib&FRw{W5KomTO7Cl0ljLV=*co5b zc8JykrkO-XEvuZ04(ArvoqCQ^LCguK&6h|$EaiQ=!^Um3p? zS|iYkO*FUeIUr6eqn*eglu7=h>bj5rM_0b@sfpHFj@3OQdB>vME5YMD}r+Mh9f zNgE92p-{9RR)}KL&}K}IJ%N-%q~t%{1hFv$PVyuj%Z?n>=0Qm2!PI5ap5ps#^=H;1YP4PRN(J}&CEbg zk4hETY|7F|M5WZAepqBgcOdu60t_cZiwIoD%s4EPNmDF4tNYN!Zsq6QjxNq%NT?2) zsssm0wrCgs&WQp2WOJicbo}P^Y{-v`bb&W^Pcb$*Mz+F#L^E>X*uHUvtt#0_g^e6C z?(V2AL{BkR9G{4f7HUBm!@OK?C1Fy|LN z4x8_nnAmgq74ujn#_Kv(;5)VWW^nWX17AxBxEJrION%3-oyCZ|&~Xzjeo}~#kj99% z2vUHf$W3`1kE3Ok=y_umOmKc_Y>fu1L402-kNNTT5Bf(afzC~kJS~f|rAE4E7D15L2C8*1P5IQ3Fm0&jSvH*E%G76Es@(#A^G0$F8B#gG6{5Sa z$YVidJuPLWizZ3S*q!&ZOS7%F@_+}((+ErqKp=TO?>F4u4nx!Wn%^`IG{3cazSKxn z;51l_MPB1L>2e{fN~h0`b~OsB-r3qKUUwOH4Kl$lHu;k53h8oNKQVJAOAb%qp{`1t ziNy0*LXkY87ho-nb!nCb)B(neWR)6to@Z468(pnUu zoSGmpZJGttst(RiL9JRQL>s+j7u160d?c{K!XIa^!IbD|;e4=$S4a9Rh9B3eR=I%U zTozM;E+9A!Qrt3_0@?$*bVsZI0HewCm+FWBnm=!r;{>T?Q=y#hb5L~qCO*q-xDxOw2GurX;oejeM128Roaf=i=v@8MtVToOgD^{+LX-AM@4**N!VK zF6x%B{Kt+eD>}TQYq#ie*xDXUh5(!L{(hG$Eh=vO%>H_g`Glrztx|JIu33^tII7n~ z6$`4m__V>ym5_u$;t#0<30l_lm!m3s{^>y=a2u=^Uya`Tn+0>SDneH=m>2fBzz>qW zepdROR_$@tPjvFK(u3w__mFP71U)peF8!kS+Q+o$F4kexBUoUzjep)0VT73yK5V(2P;5(%dmCO)=%GJ=pvO?_RK zeZH8%$aD@7(}WAh5iVLCt`R~C<)WKZV!ADYBe%St-QKFc^=E&H!_DVd?@^btA8i)p6CaBH_N!=z}zr-v6vv>2gAh%>uakvW{y-NgCG<$Apf z8-F%bCVbo+&gOs8^0Eg!S8@Pf0$xY;@@Fl)wnBR>ZqlSfF7waR)9UCtX0Wh5TM3*R ztkaI^UVTeHbdrukj@C%nE3TJ%4{QH-ewBcThu&%=J~{5sc60iBIcOiHVox~&S}$2s z9xt})3Sd!Q2pWHe^pR)hl8O^`gBnVX{ow71J*tGKJE^e`=?*8D!z8!FEd&c(9)a_#al?F4^_r@a>+TB)h>Kohko zpnbV`>ZqyoX3>^=?K`#cecjOPM2Pj7hDKfW9g0z*gL`jYG^_oX`sz;llgNgRlRmnF z)d}W=Y=xoAt)eI;L*fN-*oKYg96Z-?i`#p1y2W4RT}a#upD zwx7?2ACn;MF+`hSIBNtn8t>t2zb%+A^zXO~X2^W9&pv7#H`B;_&sx5@GE4@Wf`)6?ju)sB?Z71*{gx_rc(Y{*huU??+ z60y0(gu+EWZ06fTH;28T&4>Ro%hW5>VLYX4oB^WZA?( zC<=4UgS9bFQf{hxqA9SYZj(qrmE4C_uRiS^92eYDyQp$btm(w3q3GZK@-+~L2MSfW zQkCEx=WttDEr`sarpD5i3_P)dGPTIxqq-{2G3z-q{KiWBjNezbiA~!4>Q6);uw2t- z4-xQ-#N-`DFntQNM5|~BxX6CT#jEQfZDw9y;c7n8y0n?xV_6pzWu(HZR|Av~AA}5? z2w)>TRF7Pkr8%t%5*ZT?D1=Hr_v|))oagZ zE**>#T{@L*k1IW%fNyb5HCm^chi@k)yi%5P(Be1JUWGk$cIDAwte`lM8FddyhbGh) zx1Zpc24KyNpx8w#29jUkrTsCC?UVb^`ix4rh(IW;B8@IS`O=dV#gP6*u4-R`esUARBK@e?zhIM?bc(dGvnQhucE@eC{R0{9Nr~oU-eUaHixJ_2b`I^ z1w*x$52AmMdKLwDI1a5@ChgwWNy&3)Y;NtCrO`}>*wPWj%>}NpzWCk}>TRD5_mRtEN+& zFl)4YSjrvAXE^~0h*>knYcJiI)@GOFYegTi>Q7Es^l1TEKul%T@Za!eWzU(^bdJg5 zLHEM~AVV9fAypKRrd;f>#RyuP5oHmx6Qvq#C6!>v4>+CUNC?j*Dq9`Cd!*Q*6!l>xGZa z3J(0O-zTg$A0+Q+8p0mKY%-k8AIW~&vl#Q@isUqQ0Zk=v6mPaOEJAJ3so4rjR-eMg z#pMqj%6na#EC3+uu|Tw522|g_x|R55K`H`sq;hjXT{2Vhimfm`eDbqQ^X6?etwE%ODVLpkZ`j!s62`1t?iN9Zz2cpZSni^l^zC)pci*G7_eT#sCOz^JoT@)U zhjS#9%(6FYJF52&o~vn$MRD*x&1R4N(DSybVv|#(wgtZ>t=_Qju;Z&WYl}paPP^&-U9juc%a>IlbyxqPhFNu;Q=6pvdjV+8sVFY;SuS=B=;7ad>X-mW7dO z;Ms_U{_17>V8CV?C1E3n(nX43jxk`Osqh23{Nb{|wh)Zg(o!-vO+p>vSza^&l? z&coh4$cH}hvaxC+Q*QEj8(^x_-%;M?>7t`}DwM$1SaS|k|Ez=S7|R5ju)_sJM5#!> zaw~`+JupXG;QOZoP>Sup6mrNEc_uBV<1&qg@Ap)C2wviF#Qy*Tqy`#)nb*B7{@rMB zDgC+hFv0*1u{fl`ooC2eCRyY}32p3k)+<>Cs;}C-V$CjqA(864UbnB)y%Bj|?tVKf zeqfsz_E#4WhItnt8bn1pEVDH15hv4REXW$kM@Yh@NbDj^C{&7Sm$5opjD2+(*gx)M zvs2-FF#(po{(dbmYtFso$1`3#&kTWJ=Skv%_iP)A7eG6HpKF9I0hNGxap9l1iD~Ov z#yXV8479pokaT@%uHz-L&tK0rpsw-qRAeqhPxno6r3KVtz686-Z|qprKZq2^^n4J+ zBzowsrAt3tB?jozK+Dxp6f#I_*rnH=6*`rzsud+b)R#cckATBO4>#AmSrSdZ%$zdH z##Dtp)7UxS9SoPFy0&1>LC`G)rE$)YvMw$z$2yvZC0V_dBsx82qHIuNN+$6fcqm8_ z`2mRU#ag$$t{ylNgu?!XiZ&LSA{ph0_31s@>{mMAV^GiiK*u{{N*pougEfO$?;C*@q3T`N{tXR9~D6>@8u9EJ{thO21DGzJs1d z=x)AF&0&FL(7kkYs7yPGDxVm%9l&o7zko_cvncL?=JxEmJH z+`wSpSq(ThD6J+S9*24H* z*-0VTN0++dV9Kfk=kEPAOV|Dt{ZN-Jm?5E#F5#9D6WKUQkgX2RKLZgE(ojClha1M0 zc~B&5UO3i>Nj8$N@%)jby{NIGWhNLTK;g{Uaq}4&DP)<3hj*uFZZ>MibPH^K$5uTt%CTj=1@D&InW?;=r0ZFAGb=Z9B6U8rE>h=Miv8 zvZ4kA0@zDz#jqPes|mfjO=DMS$uhx7AY65JvJ~frp7Qbv1c2tExpsgNSRKPVRh8eu z#l#%(WdD?j=yyX*W0$(L(2n}#;x_NGpH)X4y==~@;ZRv_ri{vTXI)g9$63RW2B4no zHl)JqzkrAK6(A)T6f|dV5G!H(N+EN1+JW!q$&DMA>x&L7>d*a@R!P)99uk(}k{NTt z7Q)=HMmyOJwtS{2FI6f=vJ3}XRHkB{8e_cIR(dt8n4-R$ zDYUHmykdIsk@UJwS+GuG>nBpioG<#a0E?;a0JUvoD*Z?9G)yI_mJx%~(ZPFikZdeg zgjZDSYRT`6O#SM|JlE9b#lF-rB;`CukySRquyYZ{EvUWD6n1=v4c_sZAxpVyMxux_ z*ZLS!>D}&Jr0&XMz(6CyC>wCRG28eB4JNQVyzOXc{At@Y|I%UM{`RJ+1!=wbyjb*# zyvD9Sv$dXZ4vM)>T{OLH%QBkd&WfamBF;PbdZ2#Kckmj?!Zb~&na?+vem9tBt(R;0 zw~O%~00Mn?5EMWX9(qZ#hX6`Lb26_pL2O2cO3VUAxgrmN7)Bjie*+3FKY+= z#eeqCsKAz9U4U0tq#WRiOWCrE!!_MEfIODg_%w@o=KpfoG%nIbapZda$%lPWHiEj= z*)*Rn9T`DH!&h2Q#p{>=Z2S6LQy^(NTen*E&$QCopEt_|#;UVdjV056t3W~vu3T%W z6rQje^Em*(^zX7Q!UTW4^kPfrJn!BpIK8Y{^Dj1>pjkla*_F7GmRE&|#q1fzBr!y- zJBUSp$gsw)D0sI7!LFT~?!joBqp*H#y_YnhF(zzvImt{Y*Wvw8mcNYW0qD9#In&Y* z50_BI!230FG{P)v?+Zl)OG6jGXO?d6UpB(YbmOWQvZsK7K8VSgH?aWdT#zx>RFYs= zG?g2T=W8x0GRF`k!06&z~#MgP$ep_vuBU@%;30`up-94Wl{2#}XgShg{R zyC8P>&BtvQ!E|8suO0=Gi*)!KNM6y8(73}!yV+a;XHqrqB(CtRh!#hweokm!#2a1X(q zHy4LDgc44Aj-xrs6a$~jH3T>_(Is0j#(A(T?K!6ZDv@ur;^Z@*p?W7FiJ^$AY(?5n z57UdtZ_Wx$a~l)SaQ3i~$~PmTDB(N+zPDa^qtdjpafS1~mAG~mKA_P&bL{|@q^fSS zSV%A*seF~!d~m5t<1)A+FFr&jLOri#x+_B@SV3^kM6HUE3e9HNhl5mSreMcH$Kipv z$*d-lQeRJw(YVnqy+clStQ+T#0XLh$BIc*YD;FzI&y491Fc#CE`SEHqqIY!;qw$g>S#Kv%zeuW9R zR$&HDsB86NLm7`80zI;Dop0}*^4M1`Tec6lR z;g?-VRI2p2wxcOP#vR{m&gKWwgq#td_`wS09oOn9tY%)=jo2p9IJ)WESHHb^-o=|e zvL{T|8C7n(_kw(vRL$0MRV%(X+l$TR1fRaXxUdXDds5H_ECxjhyfL1QiifE&EyybT zd)o5Q46JTXzk}&lBA6U;h1jGO#U)4bW_Xfk5H#k2@xY-FhOMA21`j}KDNwcCtTdr% z-=P%lIbrQSSp|@K|mtQ*|xS zNLIK3=SSa|0?hfW5jpq0Hx&om5Z~TJR|FKBzDbKU_lFl8{Fb=_gxcrS&5szYM>po$ zz7xDPhRuj#Xi^m{UQ(o0TnuU);wmC26^a)=WCUM07(!TUYUIaviZDKm-eH z0W^B3I(|oAnk0g?9^zi3rJetd5BmrS`X1a=EkHCiCBpCl5=7ZBfaR8-ikZsyz3W+r zzLj2?|p$bbIOzF_%%!!IG<%601%AMp^dpe))=hLH(L=xh$?mywgODA2Y$$0T>@euA6f#%;szSF?pS48no|`XSL(Xm0d7sA=w8DmIQV zs8?~WbTUU0b$l+mM+RiA$R9+a!TC~2udYBQCPgRAy+1?yd<%T`a!*I ze}v{yLK%U%!Y{#GV8tuEu3C44__208BOmCo7EET4c&I@};%_9Wc9BYGx2`?W0lous zq6vFt_K#2g!Xjks;i+5dC>?vm?~~DKf^>q&xGWOfBlw z^{0h-n&f}JC=-;&QxYSBltvL{5XN!J!t5A}uKMy4WLUygI<;3~Wvz4S_gtVUbXh`W zAJD<;&#vkCp%O&ENs^m+Bs+I$i6QJtD5*LSHTj-B>y^nHoj`lFF$7s!9RP zcG#(=kH*N@S_}&^j4yh-%u)>k3M)orwtUZo6VDLve#*HKF35)y3rg++o)g1d8eM}r zBx#489C!)f}nf?JEH#YWgz6Qr@1Vo}uXDZ8;$xT(u= zHUFw;>Fkx(?x7p>vQ(`N4$6aF)Kju38MeX%dz73Gu~eR3TbM`k@0o{C<&)$|wXc;8imuDC}6#a4~-YfO8xQ z^i+{;3H-|W35W6p3<}GOiSi1$P{4oU^n5aU!`A@F>lfVnr}whjC2pjJvpGFO7%6=@ zZF8TGkmgPfP&0&y-AKeNT62DC+7j{hH;QL8(cSiAXm}@%yB#OzIM_-&}d!06^PJ=2r}Xhee0}Nfp-S(;ea9 z55X^aLpY~8C0eU2@|(oZE%naK+X*RIa7k_{%8@*RdH2pF3M$4>AydV#w!1$P8YoP~p*oJ7#9 zQO|+u>!yCl)hzd*PhZ*FJ9k-{Su${#ev6aygEvl&8@<%9hy$%aY&nD{dr_yR`NLhq zt8j6#quS2VzJDt2NqS0gq|QAq`x3e7KM);By)yluS&G!%#`>yzrd|3OYqi5HUy%>3 zaTM%v$R#qSESV)qb;TsB-@jqJs#RoCzVKw8AHh+GW6u4YY=iI?T-KNrEcBlfk(w_f zCBRCu5ks1wsv#C8Q+NUI1SA@&+H_jjhmTym)l=ZRD@`gHV|ulO&bP1j<7*mB5zv_* zg{4%(!Q_J?3XyP<>Uc0?TksU+#1I%X_6Y@!K>`+`R~24=UDWw7MlFXR{Dopb(&V>; zC7c`_Td=iJ*cDueRlvsBZ71mDBeQX1MofXyJ-QHTr)b}LZ+1D%J~5@YE>a%SUg&38 zoOkBqc27QJ%oe3ZP#tO@Me=Erx2_ckgfYoMi$oj3j1 z7m`HM*^uJLWE@gd$BPF-Tjvu!rww3iL=zRoV7X6$+Xj4mYnov%x&K~8SZ+|Q9?vfB zMN~A%P=QY!-CK6u(7R0n@YV+)3R2urVSRLpfa1ZTkVD&SGcW=kxq9Q}r`({f*5LG; z*6yC}?K_jR=)WTN-?X3ln4h?7dnKd4I>MAcz7sFu6lH^Nb^!zL|2)fjBY*2~Dra-O zh2tC#WVdD__JCvT-cTmh&z`FC^0ph4AK;$W)V|>}rJ3zFt^Thw`4DHH+ubiIBhsTI z-Rxyp&LH+4R)@GxG_EcGI(>x?vY~Mt7`_N6@kS@qqa3UPVN z30-odXph$8%IHg{pII4P&TG0(mUC&lxPu26eT>0x5y1&e;)$?FDIlm!v0dQavo{

    qnmJ6#U ztUG}k${{jCRWW5oUmi7(x3>#c+d64=~8GZfbeAs)voALf;#QB{b<$e7B z%fGl&BPfLmB$cy3Pmv%|#wuGOQ5K{K28Kqq<_`6hmTPT6Eab^4B$YQQJnDL$_jmU{ zodH=uby}jY`2$5=g;)ZP!H0mQ!acWH84{HKxIrOX5=xX0KDA#vJ`V3zBl9yl+_`Ki z(6gorX_w7bjmp<@fCwcy_J_cffTuV+3hZZT=|W9!XfYK(u#m3Utd(2#M7cd&JWu0C zMwR3o*73*>9Q-U=ym})`-B$e?tV+F4kwG4ZQ!L*N5xusbRuSU*7}9U%><88jZCygE zT1*V`=r@_$5n$3Dpaswr#Fd-PSxY>pnyEVdh;R@3$fZ=3(|8au8#8OB=Oy15T`1>= zy#w)es|wHcmAcxf>N<4xISmYKwkRH-^Ea!_25 zBONMOk%J$En^aXYz8AI>Qs!?Et_RP%%Nnorqo@u0DXA&x&;4XyQn7b+_vXby2Z=K? zGX(z^CXQu+Hv3G+ZYVA8-E)cTXl zDcu&Q^uQ<1@wzSmaPAW!H1QXWgsFmrKHnUTByr-~fju#q3TXxFbmP`P1@c(a59o)t z7{U3T>rr;tCVYC5j{+m>&nM3YeiMHGu{QuNS&5ib@u`w4Idn6xFObKROxq{!wVa)O zSk+Dx3BoNl0!gjBAa&SV*y`c}Z%d?X<{ke`j!Oxu_h-U$U-fC@$_R$4d4Jn5M0a4M z=VelPoT{7MKbNo`e4!g!uNTn?kt2s$S>Sv-$!33GhG=3A1@}zJ0jOi*dgJP#45m;< zJWMdYChY&ry8~9>GbuAib{#_56d4D06P5xRA;FL2wM3Dom-w7%KIb}U1_yNIDtm1g zRL4sU#{?>504y#2^2-!vf8Q3~6+4dG>N6+r$~8_5MJe62?PNS&@ei+wMRvWEmaK=P zsj1wv_DOIAipnkG$Ob6MmGK#_e+7NPa`@4Z=W^DN@<*^s@IdM&W8)Z!G$lgwdf9VR0ED_Jx0NTnbOM!mCo_9Hfio^ z$;OMK(vmgh5G z>kysq>z$V=+RtwKZWmQ%3RuOopJV;ZuQlA{)Lopyx?gf`-5Z8w?LMVi=7`!V3Uw*9 zMR4N8PxWoq1RL*7WeC4GK$#=aU>IsrC9=if2)I9Vg&O%`4ExEt zg*WaoKdQv9YJ^jDiXD%~?{15)g7VYShHc%so-U*LbS8g8L6hKrzOi+N@aN&GY(=rs zsMu?#fkwC%{o+RH)C_u2w?APf7pU%KMeE?0LiU2_^&vqYJ}BW%C@i$t8+A;c==C+D zcO1F&X8EqUEi*0avM>)-he!^V(4U5V zzim+L%B4{xf5dvJ-XxC}b~~S^c+?QA{0Kisk7u@K<#=515kIgobvlke+HlDnv@!Fv z`h1UQy3zves^~^s0(z)@P+41{N3x@s*8u;-FczTDT9M?KBQg_Qyd`8?KQWj6a+;}U zm##EHfI(5T&VedRX`Y@#&er@Ij!SV5T*%&y1hCK=_t8#;weeu=0{ktqh2jMhY|0ch zC1TGZabdjCiR`Dpv;yIHNuH)&HW%^!MJ7#HfB&8ez+N}jY|@CxmTbX)ej^nM(m1!L z5q4-9?4lycN5_b#Z=a8y*0>>9W{0@3^s&_^%IS(X-W4!~+C{VFSLGi!`YbI~Wch*? z{9dohOX2`$x(fGf_-#bo>+!cTQ5zBPSzb!}!eldNmy4HZ$pN{G1^RZ>>bim=L%r)hp{;QCKF>E#v40K6hb9bpNun)6v#6b9SgHo?F(aISpF z6!>;`s}3`Ea?P$EVd$0lJsUV(RAvQ$5LM*{6wnJ==k088ey;!Hi(|;)Cpsz98lCa(85%psgKIOd+8;SywTTmSsj3sTqXx>t#+EJ}a>UE`yHj4ImMXZ9 zHkeT*C?4g@E(*yHd#CINSW9-1$wG=}`0lW9q}RW$2nd1fnOGW)@${5&kbf5*NcK^v8i0%;79%dlD$X{j{XiCX#0WZR<6PqBcVpSwzdr`mz8SO41zR&8hz7E z=R;5{^gjO6T3Z*xedv$FoAheGSNd&e_A-XcOjNpkJI&p&-Gm(@>2yx0^4+W`RbyE9 zB?V&a^xQi|(0mdCZqF_kYyD(%aK&;oAZ>fSPr4e=~k=|vU~kpb}5`c6?FW%O05*4VnV(1DE6S-QIa&UdGz1v z2f_-%mygHxz`GK`v|-C8%_rmKOn}?`k@bhVpvsBYH^{XrZC2rg7 zsZ{FF-GM_-syo60K1iH4=~0Do*w}T+cImlKpwwIYE~$Uh4`go!?X-l|cKJ*o&Iu*l zmO%x_MFQXVn%?;b-o)BGFLbfZMfHTP0+Jy?iI*y&ZbA28;yV^jyJnF+GhesUC*&|@ zC8`J2q`L(Tw846TH9IrN=(CX z_Wu7U+YZ{G)gxegy@ zeUMZtr}?QyeahA&A1UDdt>N4rhG%lx5iM0MRAg0^!R>dnCK@?1;Q&gQQR*{s^wA0A zX|Q&_*t1EeF>V=Hx;Cs=3|O8jjvWA+o_DR?oC)f*8~jc`kKtJ20Q9I?mDx_tbjR+D zk$Bmmdp|v<<+sLDe#05WGTPLR_2G4vn8keI^#%Q8+T70HJ6TlY*0Y5<`(Cs07<;)d zGG5)_fH>7P^y53JrSf2$S$3^ z*=e?>aQ(6U2T7B#l0x!LN*Kl%JquuPCZp&||Jdh_hk`1sNequzqQOE%2WiN8`1qYl zF}rt4bjC!fiZdZMH(PIj7hp} z`N{=50M}zXF|+qqabW_?wU|Xn7aSDn13wgOy#}%F1MMh-Znu%xJ;3AGk`aT{8lteW z)2NSTVJ=1@uqR}nDPV?@w_<%3cVq0g%=TYz^mSBSKj*RVC zXjYzM+Uk_Sr=-uFWOM$!?9Z>YPx+tMfwSvJ`l%WB<(cxQhv9e57QeLTo1-&*xu1>K z%9Fo|3=1trUuZKyM3rB5kOrZtGO#gjlpLZcKP%dhf`{#Vj%3RQU!vLoOE4ojHVrd) z(rMaiHs*fx3vkXbM;fMjKKus5m&?(B1qgOv>qM4=5N6wT|0!wrSU9c3i_(@RpUlW7 zWmse67}S)ibDD)GXA@=o)fm^^j?(YD8Ryj>-~x6ITZp#~QE*7gF!kY11`^Y> zP9YK@;&eJJgFXWu`4N|Gkgy5#N&$6cOrt9y6*XE68uRfa?0p9*G1A3ao(Tb@@>kjk zH|AKGtX8EIR@D_HdQz!cNz06+hti$%^-M3{+P`EuRZ5ELE6h)BWMLY=^eprUWfdo7 zE|?)gN4OX>G%n;I;1+Ye!w{>}Et;Um&;e=%S3DhJ2LoxHcCo+j8^huG32OuXLg$-Y zMFGp7$ltYqR)1fep9j?uZNccljr!BAYCf&?*3#Dr9h0J#;J385m90XG3IA~xe#iHN zJ8A8}HzvB{>hiG&LqSCBsOuhR(%v0S$zE}g?9DGQJG_4SMNQdM!&Y#}?>TYlLiv^P z71=&LrLqXt^Kq+oFvC7;C+EC}5Azl6ug?=)tGx;AIuol&5HJ%6-LcWBr!c%|^QTsuzZRTh-fUrD$%wrX!SXh_JF2C;aBk~yyeGTxd{--k z%ePS(Q8yZIDruIjX9Yf6*r}Zpve(%FxpcXitAUh;RPC|65eH$yPo>xB7^$jA9*VXj z@;+A9^Ah1yZ_TDjv)M@^x$iO#IHLN{*&>kHOA zpKaisOpTXDLnCT48fLNIQ)tHz)~EiBIJR`Tx*3sm&LfLVw0)+1bvFN%$r|9??%`$R zFoqG*R&TTFu!sWc_$Ro)mfS_$AOk8rf-Pi1PjpabhwA4c=F|7=PIjp#@FNcS`Oew_ zOJ4gTd-pHYI92Tuy;BS|L_MAlk$kDVGID0LN!$yR1iyIJjcZZ5kMPeVR8J8FeX+Iv zH>eCdaJNTnO{QkDIc>6%ygd3=R@4AGK-{hpi%Rk|a-OW`pvaTNXaHm)+GO`Iq}=OL;y~_d3j@vopFI?IF&hC+l?N;dN|*A3Xu0DHPl?hu zy$!fieyFQzf2oGzUgW8=!kdmi%AcgmnNf1p{*(Fld{YAx@NjUmD-ig4P_x;BUX?7b zw6i7vIRfcdmGpQ>PrJG@n%ZurxY9*(Q0#Bo2;IGZGW}>Fj>zusAnV-scr9hg*}Jlw zuGgZ0_-1)c$#w_Z{5Ig;XvU? z5*m`|HX}EM>oX-q@?bwo6Y;St12#P+ppu|*NNf0`W>`4j=^kI;;ctb*etzVffg4x@ zS6R*)qY)Yx%ld+T~SpiQllo= zB)d7QT)ZSN+WZ)A)RkTPWoUPEB1IAhBHp_(D8aQg%g{vBN_?uZz$! z0+9jqckgaqjRfOz_iPz1SXf0bq8f!-LXvH^qdSeeA{lrGDFP*`0){+DO`Tu359Ys2 z!u~~Z%PnxIHqQ*LIs#gqU!J22?dMNUK!qv3pMMmFwhQ*)(tJtHR*nY+2}WRNyW?X? z+ZPXTA;`3Og>^Zp6OsN=A%IENLXwgri1ABE-XfCMi5TS(p!M)RzZVUj4+Md5`K0#8 zL$0-hW?F8X0P(wo%x?wOk{j*?c=KPGTwqfXPQw=ou8{M}Qfl=`vt*F{fvnpj%i#LV zGX`D^qnz+RneVVC5GXAp36w9V|p?{drdi#65py4bQ)y*)FKG$F{;q`Q1?y>rPn z5R#;M*%cn&Ctm()`ZgurGwsC^<+m&Fc3B0_c+M;b9(dn!2q8WTxLo(FjUcYrvQ*eS z9ja@|iP#&N;C}*KC6}KGu~m8vt#{pe;v3_5p72gga;GNOiTVk(iMKo0A-kF#+<8mu zHUHrPQ8IG;(X25Cj5&7@?ZunA1owNPOkEg#A4c;Q>$rJf@ z`1Q;kKitgT^%YO0Cj!u)fKH1AZ8K1RLbCgQ7^iYJ)$02Mvwovz19`H3^WF~$1D#GB zGtSb}ZVFKd@5P(m5&xhChW6jb^>ieNtXfq&RJ=9VEufBn{h(2TKA8o*R3DfHMOcDl zc=Dt6WSr03u`b|R8lsMWfw9!d3LpSYeK>+ubz;@gu{uLSgPGbmG zJ`47Mm@qcG%9#1Zb&!L4*XP)(y83ZU`jztzRaWi(nbEkaADSbAMe)EE&eZ>iTO^Nq z*Y_;teMLb9FoNApvlK%3&0tm0Cn zizouVarWERdPsB|QC5dMOID+}4qKkpU>LIatL3)@T43gaf9^_ei`4z6=T$2-;UV5d z>d;`dDz)o_K3bx;ST{3vBGIQrkZFnP(r-fM4&E9pSmu1do3hEAJPB8))NDdSrnkdX zOryD4N(XSG0h_BJ6y*`pUB14#q^i+(1hBd`X}neGly@0sA$=@sSa*Z}UZoko|)J;5)Bdy{K{48~0V!x76?#d@|q z3_`1s-G}TkTyTN@+-2SFv)DbFOPPsu#-Cy-2k`SLYKS$kaCx&HSF@f`XSrGqJdF4q zN9_q$4K+4WT8Szp0dRVi7Vb-b_XUX>>i(wLtB=Kv

    Zle9^Ema2v8F)IzK(-- zWB+U4ps06E9ITXgX7D9YQE{=Z(@d8;uDgUcHErXJ)!<-8$9~3(@0!1*HvN^a&b{sn zCj)K6OU2nh9Cbc%qqR;Za;CJf1B~-oE=#JIgEjRHBBaNQlUbxCqAqc7?>CEsG{Z_& zoCUF+{^NDzl&LhsduNcT2#}Vlq%ZH-uQ^%s2-91N?nP8&tCNrxS5`ptId~?S+@#|d z#kRu@{bRUsv!$S3a#M_JeRuLBdQd(aFl&HAjF3xPH(C&bu!Hr4a9EGP=fT5RlV;tn zp25%COxEvj0?$QV+B7Yg0>#yY*?0YOmfY%WuRzj-k1k+^D$&@lufcFWyF@6|O?1c! z98q0LruAi?;1>P?wA(>d$_@zXn5AfBM$WLkC4spDQsi`{i+WJZo#-nW{zN>HZgfV; z?7A9{S={dza~F~F}{b6q1)qR+dPJ=H5m zrkg+jmt}-Naln6ub%da8&JtyhbA&*#VGPxK&=SDEI%g@8Rf5xf#W5xl~_E^U-EzC>0(_@4%=Mz2*2U-h%5dgXNcH>iGIa~&SveDZsLmZH=l z`P-O7;b+T0FqaZ)Nu<+>t}rsdy-ccywG6k0jA-3MlPP%&$?*=c9h6?w=5s4bO{o3a zD`Z*1ItL{i&v)!TVi#c3uKdb3H)`wa6#D>jIo!Vl0zd}JL5H2ToDL(fTKjx57XTBN z)|Ll8zaN}Eoq~sTJK@oXUO3?}^#PTsnI>N3q2t&6Q#m~DT1KrXW>b@?RIZK1mneY2 zVVwrw~p3qn|z?;Ve+$uJoY_$-%ijcxYdh~DA?y#|ql=+qw&k*GSubd%3h zL-rXr$UUajudXfO{FQGJl{xp=Ms>YA(Ax68PNQ~@Y?2Y?ELKXw!re||J6x}mtUFq3 zYLU8bguccCmp`zAx>Fg~DBf`y^46AVUs!!vMW7^TrV6B69&GYWo~J*4 zsFqcBY)GW(DdAHcV{q~yNmp#7r?)1bM2_*a?qzl!S%=kbyo_%EQIWJBfx!_#Np}Jc zUWCvf_YlwsB`tMZK?3c*UHpr1w`oen({+&^_fuBDMrRdHP{oAtSN>I9Bi zrR_P+KO(!Fz2%^K`UIY7H&+_?OuDb>LkpvOXD|eg8v`J(CRSJHF;==5#m!j4M+Q9D z=%RCX`MC!uuxO}Pr?BSSR5!=eBOk?^V;3YZi%-gAM6vi$*7%rd#a@pEo)hNI{5~BC z=JKiNH*)#mnA(Sm6NtX-gN`8UUh1=GTjtMR*iMr#aNPw{3zq=L{A7L!=iiCT6bxpo zeATkkKNmkAEHRB{)VNIB`J1h-pabShF063Mwdn%5L2&iwW?J?HKIgNSLxXsB6?S$i za*m<>UF&AKXEI^C_uOCQuwhHwJ2D(Z_gwxU{%HhpEsnhHz9ov40LIT}(((a7R@>J}}ar|k`xgWDE1$VfdGe>KAT ztoY^1=4e?CEr9`QrWLnR_B)?mfieUXC^$*epa}milwd$R4calS17eBDw$k;wZZkwg z4<+}^9sEh-K0XX%{De7td!?9at3w*H;AN7IpMFe7?#NuZOd2h=zL3Tfu%;jNq(bk1 zb^~eb8dr%3X@;?VMfA>oCW)Un%hg7r50{PHX9!+59uAZYvtoYDAHpWk!q44f+_@QR zy|d*oBnQn%rifk8>$@tD5TbX6%rK-a2TWakB&Upt!$-quced^@XZ4abI2#FXCAAd6 ziydkw9kbJ>uw3k&B5X|1B#)srJtp|Y=NN3DAi%I6Nm&-i{k`#ZgjiNaNg4Qiy0fN< z>G(6C82IC>e8%}!m#6qMyZc4jn6$Li*ADhD>+)Eb8RHVq>Lp#d7(BN2JkcxO8`7uO z$iR~ZUy$N>HF>B|TQkn$T*k02Rp+o~nzR&ByU~4Nw~I1fHuSUBPtH{B#$(%es{LsM zmbq)agk|VkHX}`M4=ueQFZJu7bCw*XNA2Iqrd=6goiVr)b-jU@S9lG)2L4AR3N5b zw8M(LqHT?-!*ROh9?aI?&l&e{IZ=5jk6k) zk!O7Ds>JRyCdJ<*&M&8sQ!95IG-J~Da3~GaLh7=;Tou~&K zwG@&{rykfvX7lvkRI`?f`F4~*cYzf>CNc*oq6fVOKNRxhODsG5i{vpSJ3+};_Jnoe zD9$cmA0D~?i8Q;R9iN3!1F;64T+RU@Jxd4fWeBPic*cCRi~M(wCD6e{zhB>kF-K0{eRM$%Vuw_0$?Xv zDx_bPd*nk?UcHUBx#%V;WV6jjh`I&K;_QG5r#hvo^}-$P<^5l6H7k0<+z;8umX_8@ zkFuw&O4hK$m-3m^?y!(qmE!2t{!TVW9fZKA zcw-Rf!pRBX+_WCWyQMfF<(o_!*6lR!KJXOt$-?h6ufSo%(aY#o$Z5L4T0ueD|%)mjqso;AK?HZYY z$h%4M&`vB}sQXYpOt|gSaQR>d<{{rG=x!~)_puSy_AKx&Tpig!l^8}0Aj$`tA{B^> z&#Yhn`_rNKF3x{-v%AS{It|OVDMqk1TN~dR&WkdMdT4smx1wa-OcS@Iz;b)fm;#F7v2z%tVEa_OA#73cn=Yp?D3^AHK;kqSPuF5&4YKj z`OQ4c^bD}oyGPy=s6YKSf`9riajwbg6^^Y@v<}|m7D}OYd_z_VyeO=0pmlgXZ1V6W zv-4Xht!}>09>Z2&j#Rko84$oqaUxp}&iM(SRwPMfA)ItRKEoIRKNNjGY)?z(0&Cy{8Ybw(SCoe?tfc>Fsa+{`MpiU zodhn%Z9^|Ismu`NT!h+jbLzO~uXo2ZdMWyEalQyj)b7H1hMY85zlD$Wzl>-Xd(vm} zWE$+q3~>>ytitOceU!$863R5l7tMJamM<%z3F=bS@{>4~+KAzihlu66QDXC^4 ztFrKbY*C|gyyLo3H8FB%$dmyc>@&Q$Q;e|$Eo}PJ@7a@=o-=bULurwLbG#Jv*&|J- znw8vCc^s{mIAsx?bv0{{OnLmVMNhkXpsRV^>(lDh>LpNon;4v)$Cyi>t6wG9Y#Ji{ z6kv(-^WZZsdRh6zMcJbPxrdDtI32_EB2+iip7X;BnT~xXxFmmLJ>->F>oqHeFC*@7 zl`fSQMUpCy;-Rm`XX{V~tRF6RBOKmKfW$Azdk8)ninJhtDyrCJ7Z8o7vzVR7YV)XD9qsuu@O;gJv2Za<4&2IxDOX z$tJ6!F<{F$=OF0PAskX<5ZlBZ)*}2T6QB>+O!Azs5|6d|#|DZ~rM%5rU0*V_JY4*z z4d_bC(vEf!yaSDU2@<(g(&Ps0?=|OhvN=-Lu?@28t*~B?Dm-n8u2r|s5|x(EN*-?i zzZ%^7V^iVWBWO%dr2XgT_p8yj(%$b&l(e|+{x-6zRX%cACT5;ZHJKmjfmWVfq zg>HhT?{g;f_>7--RnyAUu=sTDb$+AHRfy6sOD|MeHVMmC6VFT7hC<6_O`CJs{xXY@1+-jQtmlI zNjWC<`aP%XD&qA%X6eqZ+YixlXPU;e_a>jq@NB;9gnr3*RWEbeq@C>KiYa{qG*S&} zLOHpKI zRCPmK+f5{^Z2z*}xH;-CVU7KwXP?QBj|GdCn-?+ik43l5gRmS+4?fkOgjYU>;SK{CfTLzF@^W+TG zB>V-KMrt082DEIwjMfVLppO~wYH#kYqrYCo!IKnhNQ_T*5%}6J{!WM3f(62VF=PvYx3YO@ zp64?qv5;^U&Z@Lvlml$$aJ$P)(fRCYCbvNjH*YjTwxb^`fZv1v{+3aoi&^X}+ORGy zFtv~6cBgl$Un^d{qxZA*hCf!mc( zX?uaCwQQq9RDop_#MQ|c-%IDKHVCw2lt=SpJpk6ioB-(XIw5@EqZLwFb7~TSA`6T7 zrTShzSl?i?k(Oxj0cCq5AE5%>gVtOB!UdwMhSV}M+u?D|lzj9)JB|0@`Q0|peq$Y4 z@vRaaL6JEKOdpziD&2}N}uy4S~wPQ z?Z@ePW}M}1FZUoN|ZFvd0w6p#Gma4!g*O9T93!X^k=qE4{&u5AhWtM>$cceqc$)g27T&;MdLKRcc zB8f9lM{o`jY@*l`j!vLBUdpjBKm3~hAL zXgPfWp5Ck9Nau?YspmpUfs|}3KT>(u(`mTG4&%|lu-cC{s*IS?D2bIUU0m|u%HEm&L;cAX$MxAfV3rr+jq`gRl<>zet(U?)Q$HMU%Jd+*_XdRU z^XPPm7|*4afNyo-KLeo@rRWai6ymd|jyITO>U>m@HOI7{AGR}ttm)*o_@-l(@`bo9 zpgH5(X~ao=NEZ#^C6`meK>h)uj@q~}t5N^Ah>v$`rSWIwDgk$6p3< z(++cu8$#Yl%F{K1pD!)%cW%P7y3Z!18Y{QL_!*56(dc)PuhpHXC-PS8cDn5<+LJt9 zHeYwFaWOW->_D2SKVl2ULs<~|nz`5=>KSW2clEWfQ;uv8yloMDFwDZ1H?CNgux1eW zgF6CA=*ya)kgOha$zsdm?0zObz~SGB5mDx^HNnWXF_rGU)j_IJ=C|a+1)WIw7a?sUgGDi5kz0|?( zGDiO0V5`FGf0ZIm&Ch5NmEy_U;N>2S^0ToD?CF%^Au%lT(?x?^I_tSW;|Rb~mcmD{ zQR2oxPM(aMAu{n`Q;vd0u&un|lxBhGX79AdfKILl4&khoOoLqxjhrJO1c6WB;M^Bk zDfmr)GoIkiSTC^67FkE*9sd&@-9N?72;HAM=v-e(Ncln!F5WXs$?gYp2{?!m|84y_ zA^Hnr$g0f_K8Q;RlxikBMez=akE>+vC#b$?`EPfg1zu!*-Ta(42>Y&u$Ba6i=V}k= z6>(Y|Emg~^LRpzVSIJxcfv1Wqf||}#;vguZE)GvA8emZz6cD+oZ~T-v%K`~r5%-h? zJ(C}l-cCCmm8JJDdB8COBUb}WqI%dy|6R-;haQDo_r9{=lAGQp)0`J%@iinH(}u|9 zFB=y`&<-=gKpRdU>aKEnIA+VlMOj;lr5m>t5zs{0eFTT8ej=^#k9w1fbT}(#2S4S>7MUBGZtNc2+JsXo@qx9#0F9UI-oz5!NWlsvr z;_ljvj8-kt(qC=V*~4>E^ec%FXeu%q?&|3bqZ*B|%G_ezQejOv$CYb*6sZqyaRINs zQgwY|?wE|}+7kp3v6LEw1VOUN({qCsZ0u!912dXh|~v^ z=3e}VDT)Gv#za<~H#6rW%GKb{u`&4hSb~JqU1Kxi|JJ2a!zej!qn%nZ<@4WJ_D zb0)8n`iswi>01{#zdiT?iCw^txddS+Qg23-Uerxo6#bDN7kx) zu@+9A6y%^{UIs$0Sm(w-sKGE9?hpED5)@(ef}}1}JdhIvl8CO6=UonX<%-UBcK{#F z+WidwGo39t3@qw(e|M06Mkjf?^Ol6bH}BoS(i}g%endruUzd-j_c1~RMMZ_5pP8<@ zI)>e^(7d0o*TPxHJ43;|ubiHDNi)*2vcL^UGgd{(v3+rN%o9AuS=p#HoVvwC3}y`y-c`0_egeYrduyyN{}GM{0$7Xsz|73*`=Sh{Xj%%~q$P)-n zI7&M)ArP+L59NLXd^{QLIScCvRG$Snt@mP9PSG=Zkqip4txiK|f^pO3W5immGty&^ zApWEuBv|-*N+)4;kZns(IZm*BG{!}?MWNC0;iqN*E!i7izw%-K2)q;d}EBY z%jdk66yOQRA4~X|u>ByO?T^k^T{F8}iH6Obg^gN9F`o)GLp=fSnmBSKI_M2%V=HtM zk{j7ltMCj~vdBV@lHzBO#C5st&PCw@W5ujQ9GELN5Fxl%wml0w2^wN43IO}ya4!0} zwW=q|TC>Z56*8(RVv*214xS8Q6T;bJG13aEYc?n-J0VTUlg3@jGD>wNcm=GRPceMj zdi=2-O~sC-P!HR1xwb-dLC2IF87JKeQ^7v1olpDQPwMZr?F!T%VxVnVY6Mr%lroG& zS)=+?@mzn)X6+_z*J@5Q*4Z{{HWQl6E1jh+mIO?PP5Z?4RKcw0}Sr9V3pk$SO|9KME&J%fFq zW&x@!RCO7bGMpj1nyWN$vmrGfx*)oc;-45!Kno+hZeFVabXaz3zWO2TfCltL0X%G( z%F-Zw55L~%HixV=*+k$F@JvD1`=ipzG1Wgpfd%k1=@iO&J^)Ao6_IYHmgHnB06h@p|_ z(T`!d>PLVLs3W0lN(c)ASDjyz9LFc|)0t zj8wnljJR+6dl6x*pG~S2$stYR5q_{u!NO3fUw9vli;SDTc3Tirgu7w$dg?b6)9ft5 z;e|u4`n`WHnjG%kduj4(=an(rhSqrFsY_w`GESPs7O~Mz9D67LRv(P8Zmh**>a;2l zO>v0qLFeOQVJJ9qEg}Vf)0D6=Kb8M^ZGj0uvdQDVb{V!MD3ACd3JnnuFeMTGP>lS+ zXLzJKw07WesK8K4$tP%_-(;d3*RZ>q-iPLF|6GB9;28?LrQl)6?GVj?NKA6)%39qI z1~ikBGgtRq^pfix7YD&0mt=5qt>aCKi(#%yk@Bxo08fS6f?*PXc-U%m1ShAvCo`@7 z-Gp<$K)D`2T7&qOH3s2stY+7}phqVHE$s@4P$a*AN%-`7tOf9gIDvQcwPy*>!3(jZ|SxbY{5r_hS+pU?C`*xxFp?CQtG}U*lH%3ujKP zdq*9gP*tAo@_1Ls_^!P~PPw{yEgi<(8I~j_Zxd+w5w@B-ZasoppMH;yEFPO3@|jjd zZlEuC#3RNG6o!V%0n165?0F-b06Mkw!z!aS0*{^BLk=!J{C4OX-m4--MvYC%r+JA& zPm}zRY7yAzJQ9y{y$eA*^B4&aq}w_M6ql$?acklr>Q1TSMpz9+tU*C+wTjwVq^mxx zW8WZdX?Z!i+d!xK8;I4C8dKo?=uRvHfqgfch>0u_Hz%zFxY5Wud|dkJCO$9jUdhnl zfhKVo>i3LfDh4+;rx}){2lW8e8#U$2nStDsVT_;0q4mxqyK9u*g zD(#r_w0iq!sO{K7mIYe2E+wu{n#H~WT33#KiQqzqFD?bW2|`aF1=7=1)FmF=uG+1I zOZx`7bhSqQJjl6`unIZwU0L+}RI~P{Tp|JM92kvgpeO zKIyht+;k@*`9i7BMiiDjW6$NFymQ;%4bHA+$rfqedtyAIjeIxcEZJn&Q!c1@~DnV zJayZKt>~3{V;79yyvK_`^midL^KN^2ODTqz(;d56xF@5lZBh8w_;nv_;NrVgYBDOl z1XZK**G<7OR4+t|4ah#kx+9UIJUkMMSh>82+)@ZKJRD3~Na@MEqWG(r&wr~G$V6go4ZGR<|4<`;(a({{q|;* z{;~&e^!b#R1lU@ChcM3im=ksDS>WZuIHlHUy9yEpH_9Wyg%1*i9~2Vjz=y`FlPU7j8ZT%S>z$yK?;Tz}Cn39ddKfW5REpLC5yE~{V0rrUoNmDj=k)VpMlw`z^i1x-RpKO>`mr>EtG zCM;a8C_3W9l<#G?kLWUdX%K7P85f!wxUh{j23-l4Th|b8b?-oDahPsi@8UPAUg(gZ zXhCS=J-P+910_4pV4G`V$d2`JZeGV`p0zyS0`AFdbRS^0QmCoJprYAY#k#tct%#iG63`@x|h^$`dTC8~ZBK2b5 zV*7;;x@*!yYCJ(#^R^?aTy}yL`s2|Gy<^%O-PMsZ+E<`#W6+QoTrJ@k2c96O*Qtb; zJtB-S?#$RQMt)N~11cwDT=UO&NEgOm8e|zlww>BowIt}SwYFeUmn9KiKPp> zDO*_Q`@>I7dFfFCgXA_CAYn9Q(DLG&)mWKXGx6K=*`Ot5QjvnBH8p&9whj1C$0c4S zjrrzvR-q+1G}{SP%_8_~iBPdcUKs>ar}ZbXRr zS$UBcwTLN^I2S|%$W@yHOs83i&7#;gSz)`mIrsqiRmku0$pTjh1PmD5IeL2_F)gkb z|1^k3Vc+OUMND~XT!mB6SdF*2$OB5!nfpN1j%p%5nlEyRUuL6j8h@%Ef0=pX0lF+2 zp|?z%9|9^}|9;o_(qF4oD4Bgk`V<_3~*kkm|g=Mv3vALt}3il-no6Jdk!?IM5z(H8>Rx1YFcT2G-1 z7TO_m8#+n;nT;+%YlAUf#_VbD!e4`ba&a@GMoVftblRg1$ZO+GPR&-EANib{GmVFM ziWrG;dEA61qOqrG{5Y1uSNWa=t45M7$zcxbv$_u3eB3)B6kGXx8*QN+K(aa=%;m@h z)X+)5)}d*^fYfRl?k@ANG+BdGv>;|@!{DlLxz4|%f>5dTuyx%qUgM0bek z+}W`udt}+G5tfrvAKhnEUzT}L0=#(^GtDKN(% zD9_Z&s*!Eph_#}F&`maxE#_Me(RXMn^&aZKk>FGKJPr-T|$L?GmVj2p_7>Z?&{7r2fgIraa6J zros*ZPfCzqIe>^&Lds9TwKAYA{3n|Bmi6UCJ zDrP%%;R;am5|EXuSdoBMnQ(t;6bF8yLLOS!U0<_4F-qbEYA~}u2~lYhkzB(Q2~JQ& zxF%qM^{79+b6!SSWvdRZ8;ucs1cU)U(+vxrft5hZf~|bo5}xYOh$%x6xbkTOgeGpV zCk9+4bN0>gDpn9fYmq9S3^W*xbX_c%O#4m}***d|d3#OMA*?`=VAzsLfc1VI|>iHakJA7|ABE;b2V5p975S_JDx7*)LG zgXyiXVP2fnDCUqjW|W>3m~}_1y>I~w)6^JXLJZ)k7hOR07(~4hMG$X$=b9HQ!t}r+ zQ)a_kkeX&d2_b1@;yO$y!Hp>2({#&1fGRZBDmd4O+>eAH4lmU8LwcH*S}y=4iX0vI z7aNQN2*<_&XLbWRmYLp`7=XdAe2?ACX;2cuA)_+EN@5cE1QKReg%g2#9tb{^n-F-* z7dFa|vWP4QB_LDaZ)NlMVp`!s%3lBy!uQJKvOTVY`$6@7M*#cg2D+oHo{xq2gf9As z1N(?}%J;)b%o~Gz!$!Ofjlr?cH$4l@HG-}R^mr? zz_bSiL|j0#ZOownBF>>1s6gIggU_I6b9oQbvgi2WTb*_ID~Md{)zIs zMs03w%n&juOqjHj%VPDJyYMv=EW*>5gxo>LHt>?(r~={}MaEQC8ZRKyB~1{rKZ@di z```B3DV~{9)lwQ{Bq2AMAaHexGf>(@XikmW5hZ8wC8-dxF{_!;Y$@LrdB6z=dEe=2 zms52T9skCk!8a%a-y1d-ipxx;_$>fZK&`)Rb03+lqoen5YV1%A6)moBHShcbytkUs z1|+;uKagkOiU&<6bzNYk-F@kjG!iL}SLtXB!jOo-TGDSAGmKcI)H}qN&|sIKgi zAipp9aVq9-jUK(;Q_bIQq@_^|SW!FRica7m3uOWobXrLuHIE3-N=FJ@9*D}0qK~O8 z0=0($3vl*Zrvj>3B~t-iM1(N|UHIT?g`iU-GI>o9JtztTRi`ikL7{#IdMPIQEZD`i zKzk62oKoywh<1|{v^pSMDaV4=8mMJbj}ZH2y)#w=7@$XAfCo8dTP*P~IZej)FH69`4+rg(TvM2VW3 zOB5;l#}<0vKOqgx(o&S+QY$);I(B8&IjKL^N^2qu-h80UsQ+fGyK-}82wJi$S8hGB zlC0&*cR3(7yJ@V6fiJ-gG&k)EP}4t(MxI#ZTSYvh>!7;!R8rchq09hz$1()Ag|KI> zW|{NzU+L46KRDf=TU%zqdX5MC@qwHX^GAVTKA(BVM@d;}etaL`okQ-wWcoMyH;UU+ zlm;C?+TUJmDgR(D$f`4u3E<*x)?eB1MJd5C!OO?ORg#S)BvfFUttXTTVO=c#i0fxc zPt9C5iWxT!#c>+JtFg)Xw%5R#2;6<2`ee}b6 zs4>8aA8R*(Y=I1L@1`efP&m`HnKpH_s`Crz@GBfvvt~zL>qK4{|5gJlg5ow}+&Kyx zwU7vxph1@%SaGe>HoY4?tj2%abron}IGYZFbg8a?oW-eezEp=BX8M&M+2}78&Xy~7 zZ+D&K2{FdOh~Q%e@zA_IHNo@wE6rOAU8EG#Si6w&T0G}7Sn4cz-2OAP{w(Mm#tUJ% zCsCw1D|Qhmo!mHWu@kSDB{W+-8<-CvC&i{;qYxZr(cqwyq`bCP55tzh$6c_$UvZ^8xn)nPia^DTvKPp9*OMdBlI=ySdQ`P#j#mRtC47vXC+^`A_&SG z3X>LTDjq!-zV^J{)X|oMGM*qmm2Gf4{TrSv2vT#2!FC)yYEC7_Tj`<5t74Mq%}z!0 zVTP?)ngJehGcPCY#uSBh=2Ze}h*aJ{US$Byc8qL&yu<~KR#G<)jZ1G+$%|s;R$N}u z8M`u2BrZ=X7vV{(vuqbWAyh;vJ~Jle>xZYjRDDZE_5%?v7IW+I;L5Haa{~KevLZe? zX$RTBNyW@R35*J!o5P$t^+I~2s&I)im}-#GSFYR^P{(!UgnDK^*J3~1=nxW0F?pU@ zXbXEdklFAIA{W14;-alg4uk)>#y(lLp2-#ox;|Q%hJ@r zO+79bO-yi8N3;jDavQ4+IUtyL4Hr3jQ;A*lTO6D;oQ=-~)Ou`Qw@P?|6AcX3lm@`4uz>(80V^R&fxmF8u1%U!0)40tLPcv(TheL}4 z7mTG+ry^Lj9o~zIy(I71oxUe5GIoxj*{n_ATGepz)diDvZkNd1u;eDfB1#}%1kJw9 zSQKSiQJ&caqO#rNmNTwsxn_q76r@jm0OXqn3;uAA@I&=XA}Ej{zp1 z(G=60^)H8ZsG@7b`K)w4Nx+}0k1Bi)hj-wXdKL=r4ZtD=W^>iP+2k=%MBi)THS2uI zN`SdYO1d39=GdWTEln0BpoJ>J_-&ExG|A) z;zjlEUpkg&*jZ3*Xdwo~if&kiU6rj{zfKr14RmynD`<~Mp0b1v+PI9CQNtBwrVN#S^M15qOrwU<_`m&!TFw5EV}vF{)%4qjJdeJi79ki|`4o9n^M87&#@Is@$jw95a=~E4LhZ9W+0WJoo;AmGJv5Y4b zIbnl5j(i9=wt)TS2K32!??A92fvxG@_O1Xr`qxWN6%1=n*bC zN)w11g;4nB?q_1XcBxQauSNH4nFZ+4#on3AJYYdF`cViTdmk8EkE2~T*s>!ekc*T{ zaM|wWCCda&=TMgXmS#sj)|d0*Ei2a5<|TRVc`b@Rf>kVfO3gCss_8J9FP=##&e;I< z1jSCZjB_ko&|FqL%k#OZ(pSlJxhTXu6UohrXUvE295K_7oa9ACjuawXMzkfB>^yh+ z5kf*kUtSm%!vkxuRO@)je{zliGWuVV7sCo^m*4$WHTK%@I31dyl~f3|UXiQI*MG%M zrMg*Z?FN5e?kq-+G~56TH1#oFKp7~kLX1HsvML#MspOX6VDKs-Xy%8r5AV<4fBllY zQry#|c)&18s^{Y(x`;|i6I4XEtOTjTl&xi}6;C)Tk|v~t6%{X;HI%1F^d0TXiozhk zSg;Xy0Q8`!0ZpLXVYMMS%SDN(c@49I5eYZS(0PSXGKt7@tj${qa>LjbEt0sv4P-%H zXBFROu!GNEJab(Ko-;{yqAp1$inY?mk)A2uQP!Z_BN^)?oy-)aa5VWubw~audA@N( zpjd$y&we)X5~sU1cH;ET`Z=|J-dI29*3Z)Zy*&NP>2Qce;9jj+^$H}ub3NsVJN`Fu zN5raP1r})5<=R27Y|m{+jvU_I&wfp=c+NyUpjqrK$vX|~j6P*S?kI0^asU; zCH%~FU9zW;74h`nVvx!dpNCrgUEeLrRhse8RjY~|obi;NN+1Y!3XJkj;1*AEV7{kAN zniJ7H%7~E?+6ApP28i<28Aw^iP2ANh(Z21or;Vgx5r9Mwjk2QRiAUullRz0Uk4c~mjqZ0x*(ZT=5-2Buaw0pEK>7L8oEU*p8x;=_ zDU((#BP*VoZUv?w4+#+)2& zu04FAu^h)|vCcPacNySj0ZukFudq-Y9D5v}3-Y*6#NQ6!ZIRSA%+o$+em0huMR2Of zb&KOr%{s`{Q`hJIXSG4~K(`ALMAP)QSXjovr z>iGYrJg>@w+|cWWEPX5tEl>=t^PUdlbhXQ+uG3zygh@$D zu&}T~*5zK<$G4`G)=eSqB*t?(n$^8Rv=yGdCi}(&L1^Ht^A#x&39UPB*a_FD3sJh3 z6nxK`nDr`vL0D()Ed7HhSQV316T&qx4|+>^1_GK+`|;%V-x;8cb_h^Q1%jGge(?ak z$VQQZy^y`@ULeOco~#~`ah{flFhBQw%*6UDjCKzX{H<8)(D;5xwI@EE5LVRraO-n& zG`!w(HLg`Ye5dE+pVAR_7uqMWe#>rI%qFS1X6oHfKh=~i>tQZ52WiXkr#o5>FTxxn zrVm-5oO?h&x*Cy{!T=!Iz=C+rNC|lcnCPKF{Jd42y6bu@t#}wES_3&fZXyp2Dg;7& z$%;Co-eEAjARoSZ^~D!o{K=eLC?^4bGXb2sYV9q!4g6W;>bqG3u{eL8s^(s7zWifu zkS3c(>w@c)f?UpsbQxIk8QIK8IwN;8;*j+Ndxk=aDdra)%Vclh-WGI|QYt~}Z6>Ho z5Vd1>D`C>DO<=ZSCVSVpFUC+PUAw4gw&EJIj?4MT6luks zNZ}2!`lZRzl~sswkP;0n&0*puWNO97N`J6kH*gDiCw|&w%li`!w5kdcJH%Tn$;eVg z6q!?!z;7#vy7VKIJB6MWsZ(PNU>ZEgJqE`jO5$-WS2z+kquhtl*HfqAZO~NOO2E>t z2PJ0__9kAC4%#m^JG0QBJ0)$KOiNFbV^K@h;9u_wn)Aef`JN%VuOB1WF7~+SU05i9 znAerkAsgfn2)*{wZOJMF42JWR9z4__q+;*VS_|A3F_Zv)$BL@lLBg|06%Sf;-uC)> zV?DhwUeNvQMPPMp)|CNAu*V!|laQ@Q7XxR^HgVkn0-^t<~^u3{*)|3avFD zaDX5uG}vU+*(UL(<`|!$H|dbXnrUMYygF=O5e?w|fP?R`AbUr*-y{wsr(4H2NK4%v zjs*k`f&s?^0Y`%Xf0h8i2ZR51HHEjx>+qLL)|D)1F6siT0*T1CwYR#Bg?f#}uSe?} z9r5sKP7AtbO{oJpZwd6%kbdC3JciXzXYzABlOK#Q);pEFc)+PNv>BWZO%GfWk@Kq3 zWejM`5`57#2!{eG#!6z=cL+ctXr<9RGM?}%+u8mN(17o_w?;mI;W~AQUp><$SF3xgm+2hAveAw)blLXSPC zk|w-ZA7;7KANS1z9h1FZpB-fg(A`BC0#KW=k$8*((nsXcusZcN5xEbK0vuHEHd&U8 zLiOuAip4!TYdB~Obs4-S6RCbC6X|3k{bfv~dzbZ*h@KTA_xquIA*-v4x7mAem=#us zA;fIn_j?fw7ak-u5VH22QV(KG z!QWf28)&*cC_f9M&VJvu`c`mLr}bC%)$YTwpVFO!?DHFQBpw%YY@CuRDI5>mPsN>M6fy;|TIBsJV zU=-?_qgbPEmTs0Dzw}n(iPGqClBL+vH5|R@x%NLKDZ_f=Oai1-u8()18rW91VCu7& zm`sSLXqT@*KtmK%?gB*^X({R=)p+bpwZV720l&9u2`+#acpGhu{YzF+bswHDoHm1sAM$#w#d9u>^-1;IH2$$0`2jY4pxO<-IcszQ8oxdPF6IN z0PR&54dpsrkk^*g16~PFGZfV<>*a&)wl0(chqk@i&5?C##`x&32d=q2)uSVFf^Qy6 zQVLO>K6rcewBOus%jl^?c|XN5wxKof&bhZfo;CO@+t6DkiR3xYXc>ULt^~QnS&?O3 z5mwytQWP3Gvg9BpSWdA2cJlq%hacacy?f(>dDJC+&CH>i$KDLAuG) z0If6gfe^5|s%aKj19qQewdA)Cz5ik8%7!aC-}qGEmOg-Nk(olO0sLfJ7rGFsJhs18 zl&eL~-)qPz&FmQ|wjxDw`>JUibepQ8$p%F)-7O&CJCsMti%M5^QuMyj-~cu8&o^v0 zLugctw*L?7kiz?}U2X*5KbxQaW9Za`L?rArO1OKY(*dkv4@^%_X9^g=%>{XrZ#`wP zvBVX!KbmEoMx^HWmg1pZLZKSo>YBY5muhS3jLpaekc;*=a3|l3H}@>5E9VspUN4WK ztFv6+uw7I44(10q2`12{J+HbVkb7R0(Cg_~w;Q&@WQa-|*0@AeUAGgWIXor<_RT$) zmHZzjXd_?p0tct7k%_ifXQR40IAAts84l~vMK%MBTInVO!YUa+RsY9QNA`+BZunNG zOZq2Z!g$RHFoGA?2j=$XUp39T93Fq=Y*&Ab^_=-F%@k9sgwo~GgqG+$()h?y=tvqx zeVi>Ov}j0AdlDpykXu?-JgGBU5_M14qIB9t!&y+xF}-AoDAG7J@T&h+znH_8m0YAI zVBCH3yBFm|q?aDmX+ir73v#jlkAo!N~ZWftQ5LX!5Tx<^))kwfU0CvSF?FWPT({5IP;vP zSeoV3V7$wnteCTIhQAJiUeKU-+6x)pH6R%H=61y}?7w#uG%NMe%=4awh)aDF(UjFs zCkT;DbFj!i)%l~OWNWx%czX`KVG*@NhQz2!Y(*y4yUVRo&R&T^RwYn?5pvU3q{p&p ziAZP)Q!22nL*=ND$kE!w-qIz@F2U;~$~e+uW^kv7e1U{N*Gead>OOux;w-sX40JH^@mn0D-7V<|BU2}?4kyGx(VTiI9 zs_`ooj!79R7v43c^*+Q(q8XRqC-1I>R|~W&zM)^h{tYbS+56Z2B7zasySrr?`kQ5S zi4%z$x;_J@)VJnUGnB?=rkqDZk#z`W7E#m9$hN8pZ1vZ}GrDOddt9}SWFNUu>d}(F z;#R@8N1^KYg=fdKM5GuUF)=6&AQ_x|y<3A~^?5NP??rW9#GnsZGWcu37Tr;u-`$DkI2>OT5_`1{)i~m^Y zPxDX(CVN;xkf@l^0}}&@z#p?ZGR8G}o%cB^4hDHpPPgi2{lCU&_Mc=+xx17JK4(Po zVx6(hciQ%CccEJE=;$FEFf9nfKua~MKL1y$ppZaE$+#&gOG42NmQ3o5Jd7&UI# zZjK5qzTa*u76R((Hf^k(PxM}wqK5lsN(7Hu3TLw>K*^pZuA8KJW58UAa>>&aE8f5W ze3?T_QEfqIU_}#Ws3hZuQ44o474}FRo8~vwIU2vcjFhG5G~KdtHA_3HT3Qho);G)z3QmQ z?tQni?{s}*$mnA!IWa#h8++TSGmkF7+9McEm=U@%mAs_t{_=+1MbW6^cldHIMImMD zl3_6;xL$DGau2s&GEx`pmZ@z|xztsMP0Yl#I=SB!l2%-Y!rCI=hm56(#&LSxOvmn-n-4n z^*>t+LT3QJI|_`Quc7ETS`^eZ{E3x9*YdO)?Y-z}SCWiDHE3Fpcf5Fw@AnFxL5$-U z^Pfb)NRrjE^5v44V+6%t!KLsVB?VYvApDh~s~tk%HjAMZkV^96>5He&Un&g*eL_@f z+_L@Ee!d_-J6lvUc}{+Y|G&`x|APEXekSCCToC*J>VM>CGFOYNr{-JD=d&&$wK8^z z)Mp&nEOrSbClMZZgPQV-|6#s`I{y_Bg>K>b2~6RX+=1mVr#EPoK1-^aW@@k}U(BDo zqY*^WZRVB>#VE~8PoAGHI#Y)thV2xc?V71~W(}jJ#SXf3(*r{a&Wha~-31APkz3aM zmXQr#Z&+zZ+k!tF?lCv{wb+@Pg7IoW&a2~N2?ll4nsg$9@vyZe9iPM5ex}_mZV5|g z5W0fgEuY30FAnx;!pi+jQIOmrd!tuj~^~-4YsgW)N=?f$vmOlaI z;tr*1=Q)GSYs2kHfZc;%N|KL9U3&5H2s~AdLZd@0wb+PJv(7fnwa}$BK}sspHe&by zs@z$h-a4><)ijfe^sm9mR_?8c-^qvvw+p5y3X8rh$Qd4d2_8l@53efX*=OgT8Jggx z`F9|=n)gU9AhTXIqY2T4;wIobSNd#o{LbAQ)m+(vXEPIUr9CHPspG;Zc$n;ap_klq zOck#|Lt>){GjmO4cblH+3j!~OT(>aJANusMF=ut;CXk}P@-R^^Gaq2uFJ{&%_(rPV z|6>%7Z+W?&;l0#MIg&}^|Gn~9drSvU=O0hn*`7kggCP=bvs6Po0DH9ccrO|OMe?c! z0x}~7y92JM7RP&^1{O^x#oEktrsM{w=dLq`-9%5SjWwFw7bjnkIeB)fPI-aX2sY6c zO%#jvBE_hIDy2Z}h=lYRXuc;rbQL>Gfpas-J;koslL4i!qrCa9=b=j77(c_*BE{wA5-?PDmt| zu+86YF~pIU>?N6#zllPWFH!1MUCEY}Ifvo3Oz|Sg>XfBMJ+m0IRO0D5SSMZ{+InN# z$TB_NaHk*LJP~{u4WqT$4e%I_2GnlQ6<{H&W%CHs>I5u$_i3)NK=nm1^~SA3?z_b5 zgq;@<-75+_V!ms(%BU+REyBW~R`v=)IL2NA`zzlSfG*s00K-ST$9?cv`wowF-0|cn z>MrU^_ckLm6JUr`3{m}xrTC)P5ZT83cT`Cwvr#MFXx2HWZfivMf|!MfIQs4;F%|ES zWA45-FqVVPx!V+l(PY2qIX7qNo5+aVtM+|kB+_~4uv=(|UBDbreRa-O39y7}UxJpw zxNHIxc#75KGFwVj5-?9$=}Tb-2CAtb2q$KW3-UwT0%0`SVCXn@@E`7rqqg;EStmEF zGF~JE6W`VqS<;LaiMlBLD4S{+b67plIkpI+!n|xkjin5_v|>;>VDu);r`SD zGevGp!~4cK=mLLsYc-&(Xr{eHz12^JRBLN}N2JVXr^w6gsO{k1*O9&3QFs zxjFPc@7br~jOqJoTbFY|tQpn!)a~_9bf2#PokuWP93MB5+$JELqJX>IEWtT>`Wo*&a5U>WiKcB_!$Pri8i{OR*O1>f1It@vNS zf2~#C{QrD4U?=fJHt0EZZc3!#-s@9%0e18+PFuS^`Qqt|r_WE+oL`)(*;|(R7xU+* z!~+i6X;IIN$&PLQyfBH&qD*xRx!y!)vub8o$V2h<-0)kte%6h>6F_djrNArlBo`%n z;y(0EJ5JlV%k&*Y!ga?j&5b#5@Nbou9nJSi`@ma=#=`s2qq@A}&Hui5^Wps6oA*~| zZ(out`_r}rifr&zcwxnu5An5!;hh-3ih+jru}=2))F0!D#nZ0Tx(%L&p$f{z!Ncyi zqO2OI873FxvfirSh0^epqziRRu8AnIuF3WIN*SsF$DdS-!`guVY$qZ&p;m4y?+m801}i)5R~LyZ);W#;h%kq))_Rr=q%^=V?N(AQsVF;@~7vX$Pe>4-s)s zU=#@S`SsPAJ~sf?WlN3vFhN@5?JUq`z-nL4NQ~%~v6X7T3@k7c#LwxzG>6Ps)Q)u7 zRkJ~AJ&0%XmpP5v%anKuqK1@04Jd-T20dw$5_2)7Q=V|OA{2bwSXPgf7JV3hyRpVp ztRs}n5<#($9XTOtk&9x!AkR9>;y;F3tXouJRlz_6uj`-#%m^B`jwygP2C7G5D!bC^t{P8#kb(DC)*2&K#6mTBwD{_1OT{2E#!F zMm-<|acu3CxPZAkqq9a9VZKL#Q-w~C!qU? zwpRoujrHtIRKwXRXjz-+|5dd!&y4!=uz*dcM-!@P;+50U62V7_X~?K-#sVrH!-FHO zjiQgW{WeMlg)(3sgF@Lgy3dZY4+`a=P!0;^z;*_O^6~SW5QUQEYD<+es?;(>m)OuP z{aTeN{Jl@5(mFOKw|w~>*dGJub&fs`n{n!@n!t-XG|7;ibIVq{5gKbnJPJDBvel$R znsrFBrg?QM#hqghlXIOi?t$dn7P2jn+jh&e$1KnG3{%vDZebE?OUR<`-4G&~!lY8S zMDk0cAW5{16hi0En{fE7Z%vp9`j1Mq2mlM-9}n}+x1E@yGE!pWCtvMy8Bco@-^_~) ze4sf?1dTy1ZD7Y(L>$TT=JL?`sxYOJk;9wG`PU7o0)AF03iWpOHji0;!&CM>&tlP1 zzG4kXyo!@rh!4{JnNsdKE4K;4r<#oaW)e@7$myJE9mMQ=*+ASd6pMY~ zadlvEb*bc?7+l-qZ<#{Qty;p?*jt_IMJVRhmUvqZrdI3Z`e1F<@U&Wc?18gY$I@!; zu>)hPJDJs^R_+Z~t8)}xcv=sFrL{ATR`c+p?Eq}#>dDV({wPjKYPQOdHL8cU{)!0s(u5!3L zMWL33s_C9!et-_f<>xlgL>qeC)P&5}yLB2@gJ~M8(P?&4Y3en~_1eYughA#ELU#ci zL9;CaKOwCQV1l(;XB%})9) zy_XE9JJr+GKeINIUx?LrCfl=}n#hgM+fo2<2Lmu1IM>z!69DrHlwOH~+*8b%0Zua} zH+=bmTx7Dqf?8_rGo2dAv_3h(#PvaFToSde?IH?un}5m}y# zysR_otgaT`-X0;nOQhTZ;)6rtGe-v*qoFE7FMQsE@)f&Hoj=r65^uc<-5NIN<%|fd z>Uv}f&|zV^Qn-FT6nxBqic6Tge`s@gXfIFqbXIh&lmE_5@Cu1_P z`q7a)@A5fsk28^%1A)2-Cr`+s$YDeg%}B!N9kW?Ln$fJV>hjapR9X^PwTc8)t|HrD z_8!Nw6OTd%rvnr;~jHF0dK@xnSh- z+%otj%O#gevy7O_TQ8xk(AAXElKAj6rN4-bTzow-!+QFWh_)jhtfi1w@AkNk=r2Xi zv|bXV=#)>dD>wS@Af81GP(UuNUvfA40F(mFBlR^Tu-3PqBU)9B)xo@c=HplLgg`l%z$X_|jAKhuAHz zHyD&Gw!24PJb_D*bB1+_;ujX8rgI0vc5UqJH3AP_C=;~|54U`<}-;Tky`g9CDvs3R`uzW*dvM;tOMLX&*-Llwn zT<-(lUSIAct9@~5o3dIz-6*T!hY8E?c*Hs`48}(FaN>IV%@lvCo5@C;_mb$_;?gC@*Bd3NJLEn0febwm#Sp;yJe)F?$*y$ldgg+C{!3rEw_W`a5 z-2R9iw||Dqo4KYP;{qIC zU!0!5z8I(RZx`Zzq{OI@Y{ZpMG6rB128K3%E!R_YzYP5aGCxEFa^i9}9821RV_anzfWZ zYA5YAn>A6^{JHc8`xy>uVD+?Gs}3}h8WCPUK5awO-a=Bs9&-TwgM`rclr)`LOxxu! z>9ZQX>Gjik0}59}UeXqu~bKI;K@ww$>kc7PnJ<&7=v> z))h$eH{*mUeayb<*;acHne{@~eb9|(Yn}Y*O8%UT@av;xT+#pWIF9Q^IlqC3J(y9> zdg6MS=DG%porSo3cV0L00kpqIdG8OBn>&y*Ovl?^52^SvB+*EWI|)$5kXyMKgiD&! zlqr^1(1WG~rT*vJCCes?Mz>zC$e=npJ^fcz{0UY5d53+s?{2 zbxTrN7^JQXlz&FdPYiKgiX|X{w~_!cZG7QXA!pa{@P1BO=cPK!O7&l+T$K#R?+#=N zfsqmMUu!mz0jC3@?m(Z7tLm^FHmOKi+?Wzy`Q+r}1hG%Q{PIg8azaVWBA%{4=a%n+@fGkAhL@Jt_Gq3;d(*Xhu_G?wHmwJn#Xt%}HXf2n78cKbSyThEZ?O85k+fq7X3zgB>!X!1eE)O5J+8MKMKAgT` zsmK{;CD8&Xq6A4n?YB&1OkNwNjE9R(rAP{eori3O*L9v=X?-jK*gj;|zp!CXzhNk{~%bJS!9KYyi<^rdU3s5qs)d7}#>bQkHx0;cYOa&3RkU zEHl9m^#-{MwhgZfdQxdj5mV6Y!<*KxKH`vgXV=a@DYL zmN><(D{5(B8SDK937WtPSE9&`&&w8EibSsO+--@lRfDP@bA1KL9n5Y-!rM7ace~K& z%H`4?jqIJ1DZ66ftWdWB7k35NwWQKm6Twy91?Ys$SvI_4fp%0@eW;~fg{B4yLYK?L z>wFKQbtYnVRZJ5u7n9yb|E8BQ_^Q0Jci~Wz(%aBr`3-FINXXXI-w%y=t`Q)7~+ecB`n<4T5Tl%D{ z^i|cYAhH5$`*6HId4#7*<$PKw|Ca3oALx#ub1iG6TptnWHTvofY;0r*9684=)oZG6kvAECfX&#?mwdRl2 zRkG@NS1GlBCi?GmDhg$D7(5QIP4C#p^nx`R*i7a5bx)ReOEh}9eC9hMJEYhaPBpZY zd3zn4YJ*el({QR?qEk&v(u7MWs#D`PwhWcuO4rwKab9scRBzYQz8p=wOOp}i&{nIs zRvCOL3|RS|JRs0l#|5Tg&IeTr3F7CYbI|cXE=7k~4;v^uG|L1^MtOvw{PGBI@p`|4 zX1Syh^H$@kS}9lq%*)|~B2eAQs;41OuUBBIs*1v;BwDbP!arYXy0ui;YHC~(t;{Kg z#Y|Y9N+M=QwYa<|ch5TR?zZ)COA)(ae8unTZL<_!X)4#7rK%MlkP2V5$^i?<8Y%k# z(+x9F6Hu>a!K)aIZ9#Ux)FE5C5j*q`SH}oC-j)0CsIRBIFcyDuJbgt{q;9DR~P@}Pk;R(E7Dv1uU`*;U>E-k?BwawubP1z)>}>E^)W6EM)q&*60!1FFe?x9GnDSm5XD)2GC z6W0_qr_n9ml1g$Su!@ss`{ar7gQtvk*#glN4S^6MW}4o%BOFuB{EIHsfU|mNI(e$~ z#MhuEJoxgF;35G07xqf)I9_K|my`^bjlmLeE#c1&(?+T&0ryN(PG&|D5z!=|Tpp=w zF|c_q>{+#rK)r?Z7g75IvuGmy25-7tCOm?I_;6E&WiGeP4b=2~&(k6$h18o|Wab%f zuja&*1H8Aj{Gm7s&!EHl{k=;kn3R;POhJ%QYOj~FfXjxN0qVcr9>omY`=z=m%Zlh5 zhcV+x_-V8Stjk3xG82gQL5|O()OEJhaJq$?jd%-^Zcoa`>*CF+4DAHPwN7WcBGkrl zmE7Mtif>9|jS+j%Dl(N@cu=)?97b#~@VFu){Xf>&V?BIRHp76G_EV$#g8Q{ehlyFj z6lAclvTQ#_&!pl`Z_Xbx@~uQiYrA`9_JYu-wd_!d*Vl}(81rKiR(78knLHxXr#3{e zy<;&WnZ6*D=nZH^%9$oG)UnA$N@PZtav=a-GNMZwapl2r%xQG1Uj?5X4DX0UPfd8U zyKMCoAGOFuF>lx3X!&-OP-&^#*sr-IOA@a#+MT+KadnaluH*FTQo6T!JTs<1wRiG~$mVXnt?GnoagY(7>3euw&;$hm4!*U^v&kiP%jXL< zx@X|oMVVgl8aqh79-!KO$|MLHW$b)tjn`AWrz@zNXW}7iAtoIcYqKMpb^t;~!hwxw z2FfHD#r4w4c63M5Fh{)+oI>H{pHU=Anj(r`3$$XG{(e{OpBt7g6RN^vIi7s)Sj`>p zt3J~-i$^rk+wSVSv#v;i@}$;6v2H9|%}UfGt*G@nqXahqOQ$T3E9@CGrQByMx2NLj zyR#$0$82o>%iE>x#)<(Y<``pNrhLGLhbcJrW^bRGPPX57|EVXg-~8+7KOMA#zrNEF zC)W=IK!lh#+_>%%yzSkUh$l-HjSW7}JG>YZmZ`j>!q%0D9q@7-bI$%{qbFd=O1o`D z70WsUmTfj{CX~Uko`HKx)8r7Wg;@OHh$ta|?*K3K_Ia8S_MS_qgo3n%rJ)#XUMeX1 zlX@rE!Z-enD$Ex8T_)7;%B^pM2tnfzgx(2maji59$iQIVK?1YN(-ld%RP>gO$wVt0 zZs-E8WJaJyKZ{9`DejLx(r=dE2Qs)y%H@p6BGN)}nX{2qhq^X@6GJscgO{I?I>vT1~i~#Wuk`AT44M-8H<>dG+!M7rWsc@ zCK@AZ{bSWDkkSm%!*UjhOsc$yV5KOT=HixRWom+!1~)z>kl3gITp1g|3ZWZw#2AydQn=4qBsa8lzkJpb`S=02W8(`d$kSDEzhxG7Mq)gG6 zD%vYD`!y|*vrNPb5X^~$3Cr0mZvtY;U5IAj7Yj=xsHLRFOusLFt{hBeB{ zuOYK!F^LP%#w%vp6+cnOWh&M6wjHhYoIr{A#{=9 z#;%jR+c2$p_M9nM1!tl>(~Qw*FUE2M!VZ?9vsY@270nj|Ttq31T+-6FYw~?ubchG{ zTEt=gyfx^y>g6i=0+V|rn78nOAh$yby{L7JTe6^c+gq9|?R8Wl&A^aG@VU{m5SK_m zM4@dxJ#8WRs#BG$3K>_eowlS_*&#A=c@E&iq}xTFKs8|0sNGosY{v)}W4! zX7ALCH0f+rJgjS{3JYOS8>4b7;WW-9AflshOb0~LLOfX5(`(%5^y(5`w)et0{D+nH z!A@e^rcASgs$vol+WkiF5It=5GyHeJ?J$nF5~4k4`b)uG!c4~i6G*d_3DQ#wp2wpl z&D9ERG)Lv=gFAN9CIesU9TU<8CFHi+dIRaK$IQ4{%h&{1z2JFPtNc*249U^7tu{*+ zdc#^=9_cK@V>Y0A2lrGrb|36ap?3!x7~HX205z%e%3bs(&1z*m5Gr&Aqz0TW+$6YC z*9H57{@M}Y?G{?MQId&lWRmqu=Z(?3Z3Hpe6lfNv*%_q`M(cKedUa`-^kF74WPA8* znBs<(w8qN~c2jB}HlI9H!a8M{ta`+=Sjz(ti(tr*MHI2Xqh!p6E)sGj0t|Uls-gvx zM2ev8P6*GWqDfL=b(;+!K;tyNBv%m)W8qHry|I8U6KYIUvm%Qy)RC)IpCG-Bivn0R zBwFeC3_S^zbs>zU+KmL3xfM#$g+-o)i?XRE_)>&LG)`%Bdr$LNLV*LSI3l(kq1NUq zSlPir*b%LknP=;IraS4w>bJ0Cx~`)>Ep=bXN(``y2w|NTN~Bcr2rPhn=6KiA5~ae% zQiL5*P(?slGK*!-WV5y!xbN2q>Fh3C8ixj9t5nhnOCcJ4%nbMjLrB1rp0mKind(a& z1sL3!tqD=OOq~I5EM%CL@H_H7_|To6(7c)?;$9MUFG{NyYJXwNH8b2u9<5ymOut&a z)Jqw4;fR+#LU%{{tlzUF(f{w`*d3;C&+Pd;!|Z-8t!dIE*Br*VSS~{qCB4_5uFoFA zK=L~#AsEOe{rKP++%{(!%c%m#r>Wq{XM}DHwRRM?CWN~U(8+9% z?BNYuE+`E3lzeL-;7j>Z=4H3o2f~yfv>~8<06jOxL@*|Yr^#~Rd^u3Cl)R}-U9D5a z9d}TVXStB_jqSG03}UAcYGAO_9B$&nO}uG(!%e)YeGWHqhlg!F)lxgF8yB5PZ+(Et z8=*{~0%+Eh^s|AIjawQ8-g!?~XK&pq6XE(>rmFl>C-~V7$S)rVc33sJT`y{h9qLI71)k5B07g)AAmS?<&jz|{WIeVV|~WzO!nD8lGq(ijSb8irKnuD;;&64}*; zDnsZ^9*5tCF{GHPLe^`oL=e!hM;CXw_79 zaz?%FAsLly@Sazg(T;zdnrorWETVoq{PiYgV8bgEb5iCFPw0IlYE>7HQ<~8^8&-Z*>+cFPpCj=#P_~Mp%g_|d&RwUPcua8bHQmz!UVb0~r zfcTU+u)dz#3{Lyn*tF; zY7+K7Vk|a;|Mb{Rug9?jKv+$R+J3e#n#z21$`bN(!SWT+dZEn<360^2L?(G`a#xJE zF61WAS4{yo-Nk#Y8$-3Lt?t+Boz#mH0x5#GTuDY(W5r<6nhqQ4R@3eD&3X7esz(?j z*{~Fvb#IzPaE}DZSM~v%q6`}?t@Y%Vt-_6i!~|X&s9@VIW*pSu&3bg%UDCJS`fg5a zLGV>>YlE`FwE--eFrXl}VTpCKB#=R_$tJ=lPqwjpVY6$t0~yL?uO$KBvQz;SJ!KTOE%$Z5@O#_( zm>02i17OvkwwoqgL+*P??LTu+gkC|S(}%g)Zn37Dni~!nwT_f7HP`vSTDgD&^#6L? zV|gyecWeYdomb;UK2r3H%mVzL=Oyt#uW+pKAM%_oiJ#P>r4TETQO5 z>2K>_@1X%GD~L`Lr=QAMW-@0tjL<<{?r5lnk{e-3SiKm+I;6-B#Bc4&$=D4}V1aPA z1vA-FYn?iY%z?+!tl1_Avx~n~B^w3xDRPrAy%C-a_+J%DOE(^0)v7R!oXMrY5Zy*1 z^w3}y0BLQ*_5i#nu; z*KWFM=3GIai3P)zhA$oA+BL^;_Q{Sj5xSIIF+8?XQN@lB4{?@Q*py%)l!}&Zanz1>Ka{E2%iKKiWN!o?5WLpES7gV>m z1<-1xHUkRC*v4D+mG6L`iWzws1|#Y8$WvsRm^2_1;aQXvF~$!e3)Q?TgwJn5(yg?S z>Q$=)+I*KXv?!FpH!)sFtSI+v6jo=U*X#o|?~qIO2v*1QK)|;;ADOilv8V&GMs@S! zOFqpRIa|;yV~JCU8C_$oW2Q>cXo0eT63w>~rVb=o2&qU)qXo~(5}8?9PP-$2Vkufx zl@POaPiWc$gH3C;X9ezx$G(EBece^&1>n@y->*>j3>yH_;B^*+6FlqRwC!NI(SRllY#;-9_Fc zIn*6J_01%}wA%{_)^EJLzvpoeV!-2dBG{|1CIZn^AUtN7(uX_u>uHD3+f#DEu_Pb# z4e%`KlxJ0a$7%t(yg>R~Swp{M3X6t8(FdUxMDnyq6wR0@WU}&Zrw_RZ-NH3t@9`jn z3EA@Ty0C?`Egsng>%2DD0F^4^Ha;#TZ&@^ty=Rdrwz5o$Id3hE^+ECQD`@XRLXmI( zQaVWZyNZbIAnHF&HQ=TZo}R>P$+C`mULqqet9^V;9`ArwQOE%pKF|e|LCV! z!<}Z3j*lH2K_W8#TQ;DQ<81tcH8b_X1O%S>kes^u&UoVpADd6r+sY>HSboRZ{jtgT z8r^fX7@1_{V^AQEe+&QDX>Gtu)oC1?oK38kYN|7g9f;Q>NbNZy1&_b#vWoRmc{+_3 z+qerK8vcC-1$)C_gUqvQ77g=9qhvXjigs&8iv`1+StCttDLeZCV_f%exBQOVL{5?y zS=PR2L8=*5R5I0Nj;rfu)jOJdxM#%t1pX%$6WYNtOzl}R`m0g%vRW@Qk(Y$Aa6^9n zYr|{d+nUBXi#nRj!nn%iCR` zy|yqub$4FQ zY(h`n56)%zkpGfh){xUu$lZZ2OI#Lm9J9M)8PjJ+Fock7-TFMKBu}0_8IuW5c|!99 z3xbtZ(d=`6leoysfk05($rExYau`uWGmfGUd+zij-qJLMBXTR^ccD@&L`qWyV#Ba_!gs zqC{jSd_?zZtkrW4;XW)gAJIS8zT)YWkt+msEr;G}Xb8GZSqRkdOG1f3YYP*h^Qrnf zilk0MaVAPT4G}qlSym914PWTuEnv4QrtV-(*yfzynOcrJ+oG%C1TV33IPEp?`ql2^z|+`c6b9nU zj3E+fifA;8vgBD^6sP&f2;3-P{#XfM%^nc8$N<$JcJC<)2MTx<7OQt~J1~ zdLc&}h%IUBK^c*DTn_5^`r`Eb^~E@icVSU1ii zm2^~MDH1Vn#$fJ2h~Hu`@`#Q!g?GQ!s70R}japcOR5L5Jd9?`6Civ(R-?LJO`+kLlLW$Lk(Icvwmx0BUcD4(N?wRQ_8bS_QJ+9X+3K!C=Q$@?I&1LG<)eZTt^?y}$E?PT=^YP~G4FLCozj)`1 zjvk0-OAs;|jliUI_%E$9$H`OjJu}~j^ahplNRj)6C}C1zZrv*iM-}nj%-hGh2d#gi zc&zoqwfsY1J@g*$q+f5rp{=#O)C!#)ISzqJvV5Krxk?kB-But*GhvD(GNOO5`!{FQ zpI@*p9d_CQJ9B^4%gPJ}Wqr>6X4!-Z_HZxrPoBW+IyCl=JoA97{SG{r%SiGth6|6@ zxNSx*Cuft(x?g0=`00QEYVQTP%`eg>Okpv(Hao2HLg5YsG{4OBt)#1Pb3Oj~h-B>EjEUR#$$uKNBA|UFe|-@QUtU2KTEtgl zD~d#xv8W3)H*86M4rhD(=WNYVic|gXf+k45H~TR2)yW?|y4q`Ypw%+GDeWbN?1=WF znVFf&X(si$)%_(0#>sg=4pp@D`uFyp1Ctb)tFGWUZ*VzYl{qlJA??`-2)jDmqGg^h z*T)_I75qkCaS5S0U3m2HFqZ}v7Bsgjed*-OpJ*KCFHi>0j3$+LCX8Ds2AY*#>_fF$ z8myz6vn%LDl0X0H1T?unJpa>)ckw@L-brsu(g*b}?yS!Ak1ejn5JPPWtz}IxyjJs4 zbI4)ybufguk{aQ9+7KK2iO0s;^fqKn!5hzg3DUhcnH*+%;J^Jxs~3NBc$l}>{qVQF zT=e8Mi&sE5jBfE*B-O%b z<=ezPhM(HD^VPX*S*X)4TXWZCPbsBXn#=is^^igqh5d7li-1AN}) z*NmS@8B%*~eb;@|owbv}cVyYDI1H%s{;IP=c5 zCJ)B9OSb)eXO^g!D2?~N{oMJwc}BTsw&n~Ap+vStW=+$Jb*KFmJaN`tu&vq=7{6S) zk^pDe$V8D@r&8Uew8I(woYLy|Y0}^Is4n9kiER2|OY_TfqhZ=iG-);&3D{+1bEiGH zM2|1ew?3F6SN9_CQ>3H*%31`C`E5OkwEZ^nxX#`L&fXf{UfUV;YsBTHwdWD^`$0;- z+u9L)Tt{zvDcMn#p%k0m*=$vnVdwh?ZuuTO$G%_;8G7L=jzhs{<(#F5<|1}9TXT|J z{0mBjhA|QQG#c*_q$)}Ds))>m!7_yv+QaP;qA8_Ign=iqxXzxpM>W|N!jmiI%D3(d#`@l5ecw6YIjNIEkd+kXJi1T zZQVm;4zmR_GU!@1kQ{kI$6*L?^3}e1RrZR@U<$}ojE5#7o5QMg_(y~rY{cR+i+8o` z@YsZ7MnxE%eUlebeO+X65>Pm{Hrpp>8$DobuSw0YaMnTfhxHo08$)GWRrU9-g0$WI zSU|7Fbgs`6%?K@&7(p3&8*Du$KV}5#Djz}JAiM1-GScb`sVKu?^2CUTJF(DUHD5&VCdo~Ls%fjFg zOfs4#BJ!6Lz!r(rV2WfhftSdF(b%F-Yqe;(Wbv4gGkq9mJfbQH%uv4_v$)WDwaF1( zFe37pVR`%+iJT6N3<^A6APKl0;gqYK<||ZQ?AmE&h4=)46oyeDS|em%`PF^W8_*8+ z*`x=)|YR1yXZ<%3KJ~Yn0Jwk_l@i@0d z$%0b=>5WO=$>J5<9N-LaJvpm<6Eq!I{a;FE?h_hLpfC=MwDPydp2{H=p5p>MVi2$}X zICIwQpJD!?dfX;Lb;6|{?k!u%%~^!X6Fb)`bzC106Y5c9&Nuqd6L7U4gZ6x!p>~8{)1^d=00dMtylIs&Y=uq3wxV>tm=y*hP*vo2@AOd5;e_UhA(sVfyXiOzT@ zN~@n=tcvLBh}!DatuMlBEsCpGzn+M%5HN7tLPNkdrWF(tw%)?;Vc6=?p-$b=ymm9| zw41a6~Chk1_4Ya{S_ZXT+2U@4M9l_gNOmYmpNV%C6&m8@--l{D==Z6gxHg|K%xS8KeJb_jlYXE+y>X5US0OICOB`l*b|&5K3R*xC0Z1m z048!?P3yZp>QC^Ee*4SQ#?W4BQCY=t1d9!KL6tBm3zhJ#z(K>ELMDG07|jD-$e{m@ zVn^$(&fTxA^fOjfw8n&w8k?}?W#Sj$j3YG5gu-B$BQ(YLawL*6jM&Y~&YkSDUb^G9 zu~wCQ*ePhm^kJzaqXkQ8Z#_%5mwIKcjCxr`G84t}R1z`kAN=8FtUbAV78q?=wXFjr zW_Bt&k@m_dAdNOlxMC-LFF(Yo22fhiG_$|MwfjJ1*)IB{BxP?!xo0@X?!IVjyIEu! zEHT!)(rcA`mY5@}_Vm46`7ANky3()DJ*2j6^cj@u)~m}dOAM@4uO7QCF|Ku`SJxg( zOmM#3W}iX%bGPwx0ec2Itpj zf?J7Jo00u;XN9>^lgq0A0adovTM0>G^yMmJJmD!f*`?#MQ?ON|F}*>w5s8XCXPHV?A6PRl=@%YCQnmVmmdJ)^AuDyT-umOLM52{P zl{rTLm1TG2j^^I_O-*fU+P#2hY9#UzF6j%1fb7+sCHHRUx)S{rJWIa|0)b0{X7?ZH;BI&Q>qO;m8{bE{ z{q=XZJQf#9kIu2|OHUg!zSLq~Hb2Hxg`_+_I?M9Mh*iT9?t#4yfoh*{pc<}p%`^b< zXMM&R5T%CkXMvrDMyY{|J9z64QEJ_7h*BG()P^XvK79^RYD1LT5T!OmsfDy5N^OWz z+e4HZUfLRB)dDAH@VTl$wST8LHK{jP#7?6~6q&jaw=5t4WzCkb(E~tCt=kqtYzbg} z*zE3ML)d5$bA659VvpHZU2w-NW^wsL;1)C7MCsX%$tlTM#FyNZNuc<5+;yh#aVWp# zO0r~zS`p2vvdOt~DK?kG9fpkR5O9=xE*X&vQ6w?R*}sfGI+Wq#{BNAm)S`xf#N`zF z4U{gq!Jc^tKE>WIc@CR>!XlBy+Ze7k`lwVUaEAq(SmGitQ`PTi!eg#hWkeVrFMapu ztju0DyD??8=7cVnxuDU4NoV9oJyzS4urG_-CdHg*cqxFMwU(HyNG=pr!8>Skitetm zpVS`->=9Kqm9GGvMVR3@bl}cG2sbT>B(~6=e?HOo-^iSHEn@&|ObL>N#|>!dG;>E) z`^57v$U+pk9G~~|DJemo!JBAOviEC65=NyW&rV3nvqFWIC~&SUfBK@wF$}+~Y+2O6 zy`aFq%UeotQT(>q0E8H99QQtoHIhvgVprT}LEBIwzY=CmY#B+``1r|5(sH0qE-1{+Pw z#hfM;6`a_YT{ov<8O^-D(>jVSJWj-YhEQ#c%=sP5)^;TcFd2PKqg$56kz)1P`!-+qwGCRmkE>O+dqS?&77&@d>S8LU!0!5 zz8I(RE+CUVL9`D12zE7aXTPk|q*#f9+*7QxXprx2`0@q0$YcQ!J5;TGrc+M?RuYDT zM=lI;{Tk&5K*QS=S~j2h@7ZWVU5I-U3vv%32Y34J23~z>ttrl8&T`==btKYdE>bS7 zSTS&JJ7aM2l+Jr>M|iw6I>6b)KbcBpVHWOQr|q{3aX(UGR7f`B-q^RG3E#361Of(6 z*}PD@`zZlNUiiCKHm<)u|5kR{jU2W5n5pPEV#~#`>3w@JL{dLi@E1H)&c?~79ToK+ z$u`Otxpr6@i%>cg+{`vb?na;P!LferTs`Uw3SfqqP^G>omk| z=UnV{Up#;dV~s(=Up5#djZ@x{AN?p;WcQT}^s3fhtx(^zKi8;4GV0!x?VJY|4qaF01PhJm_GP%g;H1T^{>l?a!g}hAYoG|#08%#;+;Uiuou9}Rf(|} z#Mth$17#D!X|)YwVb^7a{0 z=&CJ2%iHl z67Q)>)sRiq?T1{eLbYr5)Rx&&q1jPk0fwF4g@z3DOK%3c%`NYsXHedG&s^-*Z1e6s za(};Wj`?1*$9v?AyV>C0K0`9aH*>%{m#Zy5yd(45&;ROK;rl@)*%pf=z|qz-qdPED zJ3Of)C@Xtkx!66zZo~4fgR`xBRHF3eShwU&2XwnvVT|noL^8HMf*G@&L+IL7E3%&B z9MEGQSsV&Kw`YlrZKo{ee!0tik^sUI0D`lV`xMjk<|J?C83**(kcr%xT^!JDX$a_* zQyk3b!g33`vxWmQ5B8Q!5SD4$GrzRT9QA8A{oxP#7}I+hG7hSD8qG7dZd9jwVd62X zlIlum9{9kwV-mJcyKM;gRne+yG}-CiPS%{L<)5vhZG5EdPSnA7x)0y!KBGxLo)<6v zJiU0bR$J2=QM7{P4^rBSjQ?D)kRoZclFEDZ zl|X-6&Ds)BwU=b`H}O z(@{9P66xD<@ve?+SRu318;Uu zYg{8iFX_AS<^zyn51E@*esw`pHAv2@^@0@`9AInXNXtS(q~~24-|jY%?x28b)Y~yw z-xo%(77agm`)s!f-P7buq?#Q!!?D$t1kx?nUWiI|ha?@wPn zZ5S$w$4N$)%hj0tn33;KU)B2K(gcFVjt3o-fU8`jOf6W^BsTy0U*!4t`N{Z1_jzrt z-_2&aJ zC+WM8483_+WkYM+P&*0d3`JC^7@KTgFjJW1$SejlP|rq2qft zJ=Q%Q{|(RAaqBwnW4paKM?x#W=&|kNDz{thA0vCRbU!20(Y}rqIR%$0i>q=WnUEQu z%^r^E%`@_Tj%9^6pj6BKCw`TDMBa#rp4eWN_N5jRyDTbnu(_JS%pIG5HPIjxvkb#_ z^jQ-*pLb==?Y6eHi9P(Hp0ljWaX8US1@qp^(*I*JsZxKoTKj)YCLq7;TtW6;6I+y7zQz?Mo&vBgRhIopJplZ>_-vM+=(G{lO}fYt zmGk+Wz_^V_YP}potLUE`e5(vtX+n#QjwkNI3AG@HZyd3ifO zj`Gz~iDQ+>*dJGdK-gY?zTZM9|UAqr-NQT=;ec6KCqqkHt6M_49^MC%R$XLE!4svR9Qa`6VR;4 z5(6w8b*-+5jUnCIr+W%=Rt^Be;5d_G@)8nY!F8x8PoQi^^_=MTdIspvXSJEP@B4uL z9u@f=yQ&Jlg_7|iN%X%X?7(;!+`w)EK?&sI?EG&RzS+_&mJ52zj`R!5av((HG0Vs9 zJq15hzo2+}Zo9e!5mWt0(jj|0JLf-y)aZX*y=`ra&FAR~BfyvUA`sVaL0 zUSG{rtIA1!Nq6&V^6U>?4s-7|NqN?D0_ZN@;{=}XGyxP%s6CwZw+NnzOywe3tI&wL z2y#FZ*9)Tkz&D*#NM(kTuNfAPNR7iE$hC)kwj!q+EX|r1tjggkJFGzutuI|$WWIG8Hb|v8SBqjgjzsDRsAE$SRjrPviI^Uz zRClb!mCE_FP$HKYC3P(Md_?nT!4->Ck+WmEy7+;%dq>O{a&z z|ClWkvC`XkbF=zmVSIrudMLc*hOnVcMMbU^4%if|VNdvm*s8y-Gc6pBn^cF{Kh7s} zhV?LISNYU3!>JnN5jjm*u2M)2%CXEW&4j!z5|)=eH}Qim9W|j6qt#&oH#{(xaPu{r z90X9pX3F|06Ij;Hs?d_}u2LcGuv_(2yLWWgBM;bVr^HH`lpf5g+NA$h&!{%9+Ooo6 zy@SiF>@@zD8~8vXkyJ^`wi$QM#B#rHSZ0cT2Tkhj#%t@lc$_jtW2)$QUOv;loEG)1 zoFciW1|O2@`);um8D9T6c($~FRhJhI3}UtVQ>M>$&d^1@B-H7sGv*EhSs52kb-<#|c9KHDM8uPe4>ZwEA65sFd1HruDj zVwNWWo{!?PNyr+r0a(d!b(bvHhmOT#Vv6A;OtG@_vV3UySPLsGR=>|0LG6Fk08 z$sKg=cRTU^3K9=vGU`0(A5IZ#KJC_c-+bz~op|#Z?>gNhqmZxtvGK5XA7%d`Zalu5 z!yW-)e$)CEo{Se@j>kV>vib@r#}lKFLX{k1?Fm**EuYonu`{xd z>b1ioSv>}?uRmz|bRRYMWAcCh-*5io->gLonk#u6 zn#ddfc6T!V^7;70g6uXuGm7%~*g)FyY(9=e&V8{ibsJzaVtU*?Yw@ zNI57$e4qX1mS^z`a#lzs(rd#>Ad3}@Hovh3OwCScmI(}`F{~y+$7aIv(VS&ttussu zp2RGN&h4k~PR7svFviqhJDqVg=-RVLh~Joa46;78MgsgsvnV~Hc`l0O3)1`$c4{yALHble@47H$?6q3-_l6%|adU56CPhw@jX5COUWgp(gV(;F+rA~w z=0!sD^_PD`u&U>as?4L7XEu{W_cbz#k8fdZSdtPOi0~fWBbB z%<#I%Wo7nnvha%;pml)29TW8LOAA*9-pm{{RWZ-_FJ<2)Fls^-+prO6g(c4{D8^BV zDw8w)6;fnQe^_j!F=}Yel|7`DQ+7*7Y^fj|G$<(>w`JJyo)yr09s)5e@BtPZC<$!? z&#o^fH>KsONtr~tJC2!Z2{gf=c%%VX$wdl%vMgQ-o+&3>V*Nf+OjE8TC|gWI-S06u z1I`7}iKw)V$z?{)Xv&f^+EPw`p9`QVEk~NLVN2jL-Svkz!wC%YYgsQ0wz4TkVpi(r zwF8WY30+HT_3-L?d%I;0m~Yn|#1t^n<8bYpdIXL5AYs00&87f>1@~oQ6Y%sNfMVsz ziZH!R;i6OWv1Zfa25;NO0}Wi@u01NQZsjr$lbGRsj7hQ(_e9JT%M5?D5v_Wbt=1Qs zvud8LI0sZ|4)uck@8QX>hlf9$jQ;Y^-#=YF{i^);z4Pzb`S-&4ck29Gx__5Xzk2%r zZEK%Pc*?^-Ctv9&5m}?M=jvsXfDu#M+cayBP?8Lw0I4J0;GKwOdXeKIVq^0$E}fK4 zK#B21MuI!l+2c?{@(<5oe_@;h!9a(<{uenJKmWpS)AZTpGQsr9nY7HK)i<&EZ{o1@eF)!P4Md)0auH3kh_Op@2 zrNfKQGbj~4UAZe{^^{hDYy+kgd~U>r`Iilr;+YNaO_w6m675)%SxW#zh>;~kD==ww zIKSdg59w6o>eqXkXHUP{`cMG_2|QHDILqUsiB>-ttqEHlQStSKHF~A~)0EXutjO2C z*xDr_6IKs;s>k~OVEg>%pBzeIr*U+ z|IQySdoo}CZTW{0{J}R8{Xlz!cUYJ|OuR>Ax65JRT?ZD=gzbYh%G)Ec;FsXD78&&T z4rl%jw&XkL`(2qSC0R#?Z(uC@#s#7lSt$P*4ql zfl5Hzjsd_Gty8mEA8==u7kj^K@QAyOW~;Ye5E~n85e82;+LDx-Q~dWU8?|f_?yb7y zCq>O4!ZS4z`3OJnm|~mTtSMfgb&RJ*g~=gQNAj5EfcXp;Es-Ji&1 z_44=8WVxW*d;Q$GluEIKa#%yLz(M|mqpX7|XzgM?&)FP|?KH}Tlq4+@#g_)~$za#< z%=ppVTt?UYF&;dIgU7H&{lR1SAv}f-BbY2%WR`B-kLbqv)sF2NeK9fs8AjXrCjN}z zLM=qje`(48s~b?%zG7 z9B_94tuh5B+pa7N<_-X?xwd`Ksp{+birzxOsS(f(P%Q~!~3kR`Y>~*zdV>n7BfK)^!8}Ups z$e6rqH8cH~ksqIYO(lMGo)LUnwTy4|6W@;f?EoyZJ`CHhVYnwKO z4vq-pzzN=BUAx=>yvCTW)n4F$)aaurGETX^Acz^>msvI{C?e*3P|scn{~^b~T)Pw+-ihIQJjo+H^BBV4rD9pM8ZT6uG*t4l?R&gj#)ZC43L0x26~Bu2_)Ic&KQf4~L438gL437->w(0V z-|&U)Va#7mUM29Z$7<9% zG4S2*$7(baNh0pA8Ad*aRqi>nu^Q!J(KD%XTCh+2;~(C?Z=FVNzuI9r+t*x6(@$^# zE0a7%bA~mbs%sawRzGr~chNHEvSVbPt(1wYT)YpfQ}yJXdHNlu%0wbf=}59A%_-Ij z%Q2FODiV3Pijm~x@=6|&v&-{qVlys}NGwwQ6VMBLGD&AsM%eo(DPj+6gKVg!9wE=> zV~AJE;(C`6&C2gXU=)^B^OT7BF(a4C@g?|M_=`V`PsVLG+5q~;>@{0YXsTMj zG7tfBs=m=g%*vc9&Bc!~NPkIr4#|8oo&XaRG1fk9x5AP+zhjy80{Hjio@4QZR`J3( zVHqWZA-2!QC%#mzss-7MG2*$;P20ZW*L*!>*A02%Yr0&A#D+OOw{WWml3bqeMg5KJ z2YYEbjl+a}XCiL8aQAF8c?w-RZ|H1Dm_olXAFI@tYJAs#NFPO#1S;bj!+LZIIoU!f z59%~TaVLFg9;(2T5GB6a@~FoLP0Mka2kP>qhdxgNb$Sw@*OPGFo@{MTAW(tDmw{p1 znloidb1G&0hr9O7$U1+hz6^Xh~q zMQt#P`W_t2qPx*=FpIW+>w(-b)^~hGbLy*Cy-lIXz-3=!XY~=iTTiJ`C zJ6l=IKchD^Xh`jfK$vdRso6Y`UbBT(6L{>qHJZn)&pd!O6G*o(P3B?rm_RKiOos`n zJM7b6f`sWdx=Y>VbhW`r$|cb3tGpCp2zmRyfzyWygJw?pmAC+R)!ExOZ!XSmN+05Q z&EwPBdeO5S!4t+EDMD@HQg} z&F4^&y;u^iNXq96gT^u!A|^a$VEg4+lrt)^=w?S)^pr)^9*u}9nus};cZe1=n=?X_ zq%2Y297WLDb-(C>$6Or(>b`~rHNZZH%Dk+c_Y=Jb+jgqIh?VZn`{QKmIl(CmQ>?9W zhLH~@mkTgVwDP$SQb8P|$jRlE2~P2IQ<|0X&5ciZ$TAbgWgixRbLyZD;%necF-O=` zki&Be_2qo$_Hsr<%9UdA5uB#7NBusQU1)u~;{qI;BeiVS z7{T2$T^TZ(dzhkgB*njkmcdX*x_!dn&6*>$$R2~5eC0`KQyOc-98aA+ZFdh$4h$o^ zM43RQ1#B7cO4z_2V9ojiCh^W!d*`ukq(q|kIBXZanvk=$oL3SWztS3nwDG+&-Pg?I z#b7F@GBAbr3zm^Pmh;&PYD)q$^?N9)&b*%3Av~p{4dE$+|JP|lc*+o-veAa{6z^0A z_O#xH@D#UwV!SCVJf#Eswyq{&VPi*ULz61tl%Vn#3%CozgQNY|suzk>cF1X+o0aL?GxioL7(86WKof`$t_=g2X?t&W$&Cz$QDLMRa5%le0S<} zVYI(xtDeEpoBXwcy#ro0<4KrG7r=vu@aG}?c{lnE;m@t#dZ6%UbUVE1Q-|hTCQtWU{`)3aCP#=O^BuR`zpc30PeK`$ky=w|VX0ey%J=tI5f zHJ}geH}0!#s#!6G)U z9k5Ebw*yJebFoaAlH~O2a!hU(TsF+3wZ={b!$dBx?!K^qOJusTQ5#44-?qkz-s2dL zhWIXxeO4$;doH+bwuaC%-xZ>hyWP|M*MnZ9V*Q4(!~9ik)#_B2FEK>Nga(@>+$X@iCB@v^zW z!uEMt*oGQyZQ~wlv^6|7)M&%@$<$~Q?RhmDG8W8zTCAeeIb|y6kwmh32f#DM<}Df4 zLm9TA4BJqKZ79Pwlws>bub~W^(}pr^t<(7A%dlPJJD@`7m-LKtHp&@|fh0VWie{0& ze$HdAPS(GOPW{-?{V$eZ{$#MXK*$S*8I=Mnm;%j-8MKI+Xg#0 z8wd6@I2)mTV!UanxHG7C?$`(Z`5E!gLA`U^px(6&c2MsI_B5z>&^|HVG*sLfD(+w# zg4z6S2xj}dg4sUhiaXm@#>ixFL5w#OZyk&`lg%O#4Fla%`HN~WSoK004?g&hC=@y{ zgL2ao8)=J(B{XMb$#O1Y9zjIF&jrgluCASrGhOeZY9?*z!E&j)m@*VRkz%m{m!UBx zR30-G9p_AnlIHYd1q3qum92d)0-_p*!&f)+N9BzVw^?KL&f;T*>j&5zc_pT0mZm8n6BbgC@W9D zZ(a^PY#Zw%u~Nq}=Xd(jd3jYUAdL1O!!P$*Q)GOUoV{URQT=7 z)7Ooiy5)ypBw(3X&^wHqW6|>$=Rlt+yR@^>2R!9c-!qz7h@+pZk!-&@1B;&TRv{~j zn3R8Szwp-e2K%&LHK^`4HV^Ev7FN}IlZGi0sEur&HiM+I}=6Lz7PwRv+5}5 z%|E91a!gbDi^#^2NRLmypIn?hw+!?2>hc;7Q^fRt{ItQo#|xFV<=+s3L;=1Qi>OFh zhJ|`|8UuC;A8tD|9k;4=A#H|fIB`9~RNqjL)_#jtT|~D&7-l%_HZBI**+6^ku+au& z!S%4~g=E5oq*Q1Ucz(u-b=;f1{9ymL;+Etvh|rhTAdf0m`^C6ay^at98&oL0u^5Fa zRyLC(wzVQF<*c3FMbZHx`+@J)`C43i>&U4x_ABRt1!c1Tt)k#_HV>_;dSg^Yd}Zqo ziiDkBzX{V+zn_q6kuY+4{pN_!j8M%CGphI1Y0QK6%4|?F*vMe&o$(e{bhWjpK9o{JxfpJ1#?U`4OTOFAsKD}Fn-ys8&+MjryN5Mkpo3jg_%=zZ*;sc7DC(dZxJ=Nrk z`R43OkFIeww)Vx4wB?X&<%`*`n43p_DF^xVdb*8khR3QrRNn|OPZ&8%L=n4wp=SAL z6)qI`rG;aPAxGc5ef!O;i=R%fu3lZ9o!(r&ee=`V^~L$co14qiSIrKyrdkVidPRC< z<2eiytd5Re%d$Do7!pwxO5}Rv$8OqAHXkW5f<8vegsPdy(>9Wwk|kIxelED=3InI` z((LX=Zt9iO%|On^j`610MrPfe6^J}DJF4P=jXlv0j)l-c98@ zyof=O3O(*--(Bb8H7*p$_?EIyN?<^dv$xHNj{g(?G16DdzaIamLyeK%SNhLmEhhik z=$H@C;Z%EdrY2v&gSrC9WR)rU9)%W{wnr|_P^O-Xq%jGCVY3^Zif?qb`ep9se&P3yTBO^f#*9xJ?P+?kiQ7CY4S+FKwC6;WT83Vcpz4?Bjs(?=KHMeA z9gZ`aO5+M2(8Jp0bWi^tpoV)dTQz_Z`ldQ$nGIQH9vTi=X3z#u!jA+>IMrj;k|xaB z@U?v$4tSn6iXkg*yzZ7pG1SC68e}(miFh{Oh^sao$+kls!sVA#%6pObL4~ug_)+0H ztZXiIY==!kYykf7Jvd0Q-RL(+vDR-rkQ6H<%lm~+y|b_Ejo;V~2YzGPz;6bA(=?+` zp5JtKJ-{q~T$tqnre%O>8DLrln3e&ir4PLZm=>oEFf9)wyB`ru%Yz0pn1Uz^R$AtO zuFKds^4mFu@y33tp$`K>O!IdKgqV&tln@_k^kN%Ih*#~Al<6Hxhz}*i2lj+*C?Rgz zBjHU0Ld>Avxnm#r=V!z}2ldWvgL>CC*g?G;*wdiiLHopb(|`~&AjDuBqLBP;h(h|D zqL4mh`D2VPz8CHd&mQctZQHhO+qUgJwr$(CZQJH|Y(Dw_o|AK4+;3L8+SRK~(_BeE zwGGA)iWC?-X`S9#_UpUz5DlYfyIsONy*|Ly1v)a0V8Bz#X0phuuC&WXPx1FpXbH>p$YJ}>;d z2N{Rqwxjn!BTz6Wf>XjnF7Ticp_4q_Ds_f6^W;n+)d$(yNzX0_aT3Z&+EfZ+z*R-a!A-s@Ea2C zeAX65`sx!$^09{d5}v5a6xqp}#(LOhFXMoy6jboE;DDgA5YF}1#(taWx6j<_x7X@> zc6``Id^xdaXIGyKTIM6ag4Yk}&OP&^&X0?>2<$O~P(lpda5QV`nWX(QAc$>b6yaoGZ!h=p+v(ES;qooH9nMzcEl&n55M=@or>bq z{ElAJBuhQpir_4Co#a+p`~fC6S3>DPZ%Q!uAk$b~u%o(ZVsBk@51zUfu*JeVz{cgN zX9FH=5->_&SRWM-!vNkmpXKlXqhqE3BfLsw=AD^*#;R|ENCt_6^qzb6WQy=vl?^pv z8~=3K5Nzp5R}OQ+!EeI>*K7W4Xm2yX_2A>H;FcPDp7Og24vYaxA@nBSVg)tTeQ5w$ zSO4qxNY%v>7;B1t#*T9^&*2`Q{Mi15aJOd3x}Z`cIu3AtMVcWv1N z>j`Q1IqadVu}Mi`zzgzQ;}l-!i8@SYI#Y#=wGSAuxbk*j*t`+z8?e~=r~!9E+MpPq z&Q$K3sL+3wM(Vw`zOLpgbiV>?s|U>xZGN=3AcfGSlcnUHKO*>g+PYHL<3kg1z>F}3 z4>}p(_9@VlpK;LqstAU?d8#~Mu_+I-r@~*2hd6pzPAAS%8Th}X2S_%7l7-BTh9wx) znsdKMOu!EuvPgm*sgHHTg=aryx+t-5ON%+|uvOS`tDqgAuDnjN)5xhxTj#zlds$KCpSupPlKwG!O55kRno1H$D>DW9Y)aooCgHm)E#G?-WBVBI z9Dx~9@)K2F%i}C@rf%dbj*<0BuqDnjw!wGEzx+b^yU;4%g75Gt^2yjH?}F?0sk4@# ze>>-X$NmB1u{sBOt?Z}vkJf@X)S}934p3O@mW5>nr8QUrObbY@(TZzvSIOErKv3$m z@iAWOJlQ&_g+ZO-E@KIfh3Vaxn_}kz?1j!B&r!3yE7xk=&|mKy zMs7AXvcIM4-zT%*?P9rRLV2l=bQfT2)-A{q`$l57;h;PbCYDYGuT+*kOv46u3=vaK zJl;RjkC7v|BsYAPCJt|@UxXH1#RM+&Y%LN6wBc&?_c}r@3V=%&(n2mS9|a|xb7!J_ zmLh`ssft*o$2G;_aV990I^pT2fn9Z3_4K*rwTV_Fo~FiiU*U?to_Da;ieIs-TFY$kY1&*6xmI5Oplo7<*0ADA=SJU5>sZF=aP&GC7s(vuLH3IwQf5* z^?o)f0C`v=0cq2YZ?))vN#_8%+K)oo9TWG3?UC}VF9y&`Jfll zrGxO@mx;G>GJs7q{ebq*v~b90GxL)R*>kZBY5lBEF9y<#b2krCX)3ke1wz>Sk2@0~ z(L~Jbd?)Uu{f5>_kSqQ*`TP6$WD{wZ2c6sZb_;Bt`t%x{{JWd-lqSz|VRkc5&FuU= zA^dG2@H;F&ueyKh9_(Vw`h?&2`pt;i{Tnyu>#t+fq2qZMxM$embMnr8eZdzU;kRGn zx1VzMCx0IoDE=-M*utX>-ooRI!TGZ#PwXyYFehM_+z*1UAWZP-^u;a&eV6>n<-;;r ze6kALYk3S+KB_u=NENS}30@ciyO-MD#OpC-O>Ug=#PGEu)3Z}lU0Ye$p#*Rgad+MvytM9RR!RID3Pb=8NBFPyhq%6D%>}BOP`)4*%CxAi9 zp&oM1UP=Q>@1#r#S;si@bh^%)lr$4Gax_`1(^d9-XP}&QtjSz>0~vs$3aJ(>XjEgp ze%GG{jQQ;IU4<9{^3W2S>7s_`+$_^dD`8T~o$Kix#==U~|5d^cnlgik*4P~0(LUas zS)^B}HM|w0dsB;+0LW>C|Mw#84#u1@AFZUnbZ*Et9RlUnb@tLTh&{|2=43fBTbHM+ zlOIhX8qDJeIVp#bdY)^N@nb)rL}W4;0`I3Gs`C|0{>zp3)o1VDz@IJq^YW6${}Q4H z!vOIVq!v^9y%+{=-w5r2@_?DTEsLOOQr~fXkdM^tB+4@#NT_qBN+$-+K_LBXx0w#i zl=``yt1{iUC$FfiD1Z5JC8<2LjAgD1!irU73pey&Uw|Mwl32d*`L-T zgHA4aGE`snA6dn;s0>prR!sVy4<{%5H9Fd9x(H#K#!&|R>o#6=gJoOL+|-AJ4r!`J zcc*{{L{d#Vvsy_~4ps?}z$V7Am{ql!;Bg0p{ntY%chv;y?L2`UIv2x=Y0{tnT@Syo z6_g!oYQk0UkC`^%sh)MsxvyMtbsahr+ob(o-5?wme#+PRx&p=cLy@FELRpmTP)m~WLR!CZl&!2L zYV;QEt<`+`jB7)i5FVQ54DQ>&yj71gQY+*<;%zq4nf~+oP}H6Z-j!0MQ#b@A?ITfp z+|!j%bV@zy`-M?^MA7CLJ;jlIU%IFIML~&v?fTD5WyJJ~udZjGtthG))XXwbn_$Ep z+MGJ1Lz;-@PUuFT@m*lpBggji~+qthA zt(fPNJ4)KB4`38tbb3vkxEI@7{{ zid6me(MG?Wbw;WzU41fHo62SN`aU|VSM>W5dexU(V4gKec%B1`_8x*(tkelJ=ui$c zrx=?~x0$q>yepml28W!M zPkEoiy1K^fOka!s(mMfKfttu8Xen;JYw7(!h9P4f+fr7a(nuyq)hjF<_o8Pu9Lr&q z7r6vYySd2O=pke{$GYXq>`c6j75bLDi49-FTSCJ;OGE#OYSWDKJ14Po8KW&eQ8n!Y zu;cII7-%OyB{q$D-^j7>*pZBJY>f+qz4n@%#Ot+uW>S(Ie?~L~1iJ9_l=m#{KjOjoW z<^3AK&k(!nrEfaEK=Iz}lB_Z5y}v7wn_+m7vU|c#qyNYdhC}NGF|x0`F*;9UXz_%i z#Wi?az|hwEy2H@YZDD9#<6e&EZrZ}o*0#k(v&Uo@i#^xwz9#tp#J-)C@2+f>?>@V_ zsoaXauvEIw^D*tRn8ji;kEOGj#oo6uI!~u_nsxc6XBZ>w&syNrUVuvu-|)GDJ4F4U zG~N4dalEuTce_W@^je|gsM60M>8r+CMAB*CHk=uFx!EC_5K6vTiDlp;&7Gmb)B~gF z0fk_-BR~gY5L(_MLRX6t=pwYcb^SK4$q0GMBd)J+#^?T9D4*}XIiFBdNxTSm^!agk zUQ~x(<%Sj3ke0!mS(n^2znG49I#iT$Qx)^U*z_PJ8P6(wUj#3*xmKNkn3b2PSXaz| zpoS(S9hjT!ObPrafl~^~Q*Kj2Ex)lcF#txL1@?zqB9utxb)+LOgc4sVu~F5l(K z*<+S}G_CdKD2T37g^E`BDbIIxQ>Pp-Wddp;%NYIe0?=-m+*2c*vj3ehw|{V(GUx;8_+fKK(n`UJ{&BS(|dDiy+{}+g1JI}QI z@T>#d+wm|?Y}8eePj$K9$llr4Y2)U&qSw!E|9;BU(pO|GrX(Hn#m4p&qtNp|cyJL?SfRP(6*|9E2;cTz4 zmC-;KL(@^L4rhilmPT4^qf9i_Mb{0|w7hYSFiAMmp)l5}0q9R6pjU8CceuK`=qbku zaQ~~Oh%AaWsHj9i9t)**$Iy^=wkorz29JeOjy$s2hnlCkJe}`4d zGlZ|+;vCT1cWv|gwnjl(DFS>-t9}{bh4w)7X+L+UZ@q>K0JJ^IWql$)F?$HKD*?@svZLN-frGS8X)+i4f_zjz_`A7Esh!BBX`9 zbp2b2f6EJ0$v!~u{@xs#`?MTRQ|&iSnd6TAx%{qhEgLnniNk8$vQxG1qZFI*Tj%Yt zIPC!*HG+_dYNxc>%OV+6S_ETuAkQ{6)j~3l1wS|tmiz0V5SQpa;>Uc+^|*Mm$wW`P z-*z4^%X}^5WsJ}W%W10E^s&`qdo^)+fw6t03*+9Mw7z9BoO4|2)i@w>ty(-1 zw}bc1bvQABFg|Ladz4LRwD`p~Jltw?mwVc~3>}x?LYQqw2;&hOKCngkK?hSG4Pp#l zunr$l5*7R%D!7DC7CF%@4kRN!tBs@K z&IB;V-3%F1xP|9qU=ED+L%2m|yzx2&0B!s<1I6Vc7^nz{^*$~<<|zKbM2%PcgNZp% z{_nc{-|*ksTWS+tmb3td=N*vc?`-PnmTsIY#=`EP!7d(=yy&9{#Qt++yWhP6pY@z_&bO^i?j9Q{)rt}UBrqzsQl9|MhkvDVkI+8Bxe@);d%W^=$c}kQ}>53P5hBxv8nCsn6#*<^48q#($3Q zck%0W;>=t>$M-pmT>s~!il5)};UrC;_kC#NOn$ca?F;_=x&S}cGDJn=>);yZEIW$z z##fGtC(rvlJ45Z~mDn;b6=7y$Lwtk|2rm1)`xyWvQp7%%SYccxvl;h70$-ReGQnu%%B!B&k z@aiHq>{{Aek@+uBlfpW^PbSJ=mVuu%)SzUi-6cZFKOyoSj$5g~pY3er8t8C{7j;gd$K zi<=EY=)@ys=a}`RzKrw_DAb8Uks0P5qK)PnSCnzxKwhOMBJwudNXwj7p9i!hOOCC% z&rIl$>Jp)I&-xV+h>Rz(GvA884UvyERs?OOZGSdwDy_cLbgNP1QtP! zN&G_UsL01dd2DQHP4+cTp@5#(rnpE;R`G)e{O;3kBz$j6a_?DAy&hgk4f_dPe}hkv zTjqvp;F?__Wc3kNV2wmuF6ZD1oj)_nM0PiA4|}$rbU%2~u-$)z+AzTnsq(}*(JDXX~qDH0%)67$M@hY!=ehoG~wR5#d7GX~JilrU2bxf#a~FXZq{-(!1L)&+c|r z0yM{iMLR83thgtQCyP$F5U za~pX$w9C{p1K%zD-WmkUzt%=-e>Dp2(XRL%hiYofa)hBbZUcNNvq%os8&4sb! zB>cjV`DVuV<6G)?5+0N(XSm!oJA#@>-^}<4IgrZ+_n&eR$W#-djO8?Y(KRxOv$LSz z#qH-akndo#`F1p26C=-Vmn={ApC|)C5&uot<}#S{5|Qv^?-WqWDIN@7iC!xse$b4y z5yS}7dU?$JwNpvwg zDe^VxBn0EaI`d&PZ9aV(8hQ;&R?wmY4wzepBt?XBfA9=jZMzqD!+Epuj+Z5z?i>as zay&yEk?vpVREk;-8T7aK;)z{S$z`7riTJU2JjFy}F{@ybP^6!4m~0=k&lD|z3wL(*w7ekJ{NLI=LW-}89babdvJ9fbN`uU$QAZC42uyhd zogq&u-By0V{x@P6P8X5VT#|-XmA|Iv*fPw(Ep)X}Bmp022*UEQ0VY%)xk<~W-H>H4 z6%H+6An(Cu@R(y2JS)sazxIg>oRVykVsiW|ztd?NNPRjl zbeMBYe*U6LR4i8!+rBkzRQjgh!R$j*pO_gVxEPsaD=X-?yMzbj{073nBK88nLz{Np zH9g+e{{oF{VO>UxRx_-&ShBw7OJH~MgUvBzBP4~Cr0MBEG!clyGzt)i(H)?>EbgUr z+o$}xbd8myR7!B!J=2`C02cJqAup$&mly|v8F$cK*9Ff!G-LznxKx|m<48e@O$B<> zpGYki;OLlnDUuuB7o7)BR|b8eR=FQWZx4@y#k%hgH|{i(2Y^*??bcf=cE8){2;jw^ zjbOCLgYK{4{GL7DlX-)?yr>c;w|LOSP4)^=7$y`gzCn7!%VCS#X-~iv*V0}Ib0J83 zU1vfUw>!=SByTqz3RFI?IuwMx-gL=md(9HUb_AlmmfU2M@D3Qn!+%Sr;1yuVm)L2^ zN_s61hCfO@m#(s2FeG1Oqhwq3@s%8mMT&QBG1Pq?zWdWFp&*5o?!rj#KpOSnGkXn` zeqtxE6Epc47So5nhenYThWm^qf|TCJ?x2m8@zdw|Su{_&$I71J<6+3Z*=ZfT8EnUZ zfdtVrITJu8fr2UFD3J{|Z$^M+qTDn>p}^P622Vg_ypxH5O92W}1ZoVq!{{gw3D*?t zcRv_5!S6iHyMLX~E(Dt2np-mtW;%Xadc&_m+{Qb0^}G>XzXwqA=^}~r0Lo!}4D2HUc^{Y+rJp`Mn%@AU)FWg6STNQ{E zkFY$K=H)TKrr}qFeJ2lu_H^me6JFRn6_iQNYm8s zwX#(o!bIx7(8lZbh--_1{-Zkiuopo**qi@bE1;2)G4L*rh7B;h2kB#Rwqo7NY?HF{Z3CWLagev=Kib8% zI4tgAn0~xWB|Z$g{5FXZNH;pBl%w@K!zu6OMGvFu=S=STl3vB;Mg9dK zhXp`IiMMyRH;2URC(?72A%>UIr%hr+kn%H2Asf7`u7t{Vw6&+dS$K?wb`zM=d3!5p|2J2OL_J9ZMo)5V$m z&&$d{!!8OmtrI9c+LW!d+po(TfNQdC2FJuZk-u5~ zRF?%-F!YGJ#sa9hW}nJ{CDj#36XxFOF`i_IW~69jFTHA>lIk;#zz!tcd0-Nt zdDvnya9N{MUq0&y+1kQZn^j6*&7JpuuUJB!mRIiilRYdP4k0@6@e_y+9Ktl;xajyP zIWOX4r>hP`C(i#3l-LpsQ3HDwjTrdGB$lrlRiRJw&Pfcf~{kPhBI#>%5C7~I^ZQAd~3J^k)U(AN*Fz9NudF3)# zKb`6JHy)*UO~UY7UN;);q_5s~VJFIR2bunQQEyA*0r1bY77j+!P)c9b0w;|@LU6*w zgOn7CaiILem=P+D2nzdYN4&1O$WM2CG-END+9s79xJ*S`dGD2dl4c zhX3F#>}lJI_evi)24( zv4Y-pm)(|qBiBr2XRf>m?M9}w8uDjLb(}4Nva+uGSZ2%XW1+4n5XC}JW1z0wuu;`+ z6~teb?TQAq?C~+}Bb7BG6u7IZB};A!IA_LNZ?G0uqV>AmOH!C56NTchxHEqtUY5@9 zdD|%aYbbkaCxO?XtSTM3FJo=+xB!{d{t8Wg;%p#CdTpeJYi~x$x8aP@2{pzhS3{@v z(KdkZRbtUrthBW&LQ^rn`WBq%z3G2q9V(_1_1})&`{OD76S^tXKUxRT3`E97ULSgU zbXW)j9|7qii-?Y8o1?`UT%!s?IKqEFqRCWLD=xe*xQ^gVO$EYxXJi)KHHbzLGv-*4 zi=CTOxcc;cq*=x?NDd;e^nY5K4!wMRKjZ`4zdr(ae~+8+j@7Kcw}aZ$kR+la(i5ts zV&h7VA;A<3QGb^gHngwO-9t8MCl7PYG&Ki?)npg(dtLVqfDdS>g|WCPey8WzbqF5d z0-X#>Dsz=DB4_tzIoS{JKCp*Aq-c%ijlln{Dzgk7qD2POe+(Y7017YuzVQ3=r5oNq z24LqOGVmp}+nCcS4rTmFcr7;m)%R%7g zeoCrIvB+VEj!2ZNFcw>UiSBkzQ+eEW1m*d-!DQyC*TC=Z~vCMUF~5X6wV5R~?A zZphXaTtUZNoQ0Y9>CLDusb)BlDxnO>$VOhk9XwE9hUs3c;LO%E>@A70(HHb4U*wMX zMz&}df-&dnuHn_5ffegepLC~I01({CHLNmz?R3PyA2z|b?-aET1x|Y)eS2ga7hM`5*}Wucg!haMaysQ-oa0?bzoaNnR+Nvb=4h>SQs^q7Tlh__Xt5MSl zZ=6>Cxh%FRm}=I}HCO1jIv(-PU3aZN;%>C(>oC**JJ9UhezBt>@ZdF|QU4ma!C|*v z*r6}$7?8akKbQ=O|w2{k~x#k1=Iit#dlK&*eE}bqvi_{gjcu_R3lr=;QvU;58Uxy}Q0|CXF z&n$c8YX0K*00IA_t*wnok5A-THu}C51iCq>`K<#n+!NfI%mzfwu+tuw5{B(OI0ZkF(Xds7_Cv zw#U{%y({u97uqer&aRB`(oWC2)sOv_v1{wfU0uCK@TYoXbXiL z&iY;aN_z)Y)-tY8(cTYj(J|5b^3d zilMI2*k(EiwlC{@cC|e6Wyp}_sQ_h_L7XLbD+dRxXYYQa#`9m1=WQ#d@`=ttRlHqB zrg!tqu@Zye9wDbGsNlO(9GKpKojz?DePALcff*{(rka7xC=vZ@|9twe`8bBQmT@r7 zU~3Xfoy*jG8!pUKFxJTt{uN=y-@%ojen4^MH`yU<-O{o=DOAZKD>rUDpf_`}%STr4 zM84DZ<^1jaPD@)B3Vf(W;IMYvsqZ00uQYqk69`ooR-#rG4HZ3Vk++$2PT~i$`PFh; z?*OJs3T1`WCe{Vd1A6#7lh)-`JJi7fRpDtoc~RBxi-XY*?# zHRbNrjOb66uu>uulw#j_+*r+M?gJXSI%u%}(#vo_#1^h7{%`$9?6++#@h=n#mm+67~gXZ>*0{8go+ zAh9j?0pGgedfeRP0rGU4ghAvXzy5kS9HXzr?%Q=TOO7rmMPu{NGWP+g$d*FdL94_< zw=XxBRaJSN$i417wVR`t1TaWp$XY%JT-a8``@n^A%pa5jB@zl*{Z%umIQ_6DLZO)t zT<}k6mUfArc;707oQ^Qe+PNP-Rb3)Sx&Z>KT5x7dg*(3tq$2M@>eWJxuAHHSAmXA% zL4RN&wNA{D>P$CCps65=paSI_;mi?O%aD|5k>Yptmc?^B@6mq!i|{uWb)IM-Q_woK z(5evE;Nx=SS+-^7*{IokZ#$BzOQ!|YYmg#qNVA}^(;}tffLg?DOG9s3O+`|DP)I2C zt&++-;c6&DZUIbq?k>_~uAU^W_x~ZhmV}of?KaagjHjBkfR9ZJHSjm52TQS34_VKQ zsK(yvy0*SzEhT$tZwR-105IA zT6?S?Z1^smlh$&Z7?eRHMT|0GciEIx8MOsi<*X?=X2?{nJD6p|{)j?r7<{co7J$h8 z40Ct6Hu+PoormJgWGMnhV(SAGX1Yfjs=07-!;8w$;<7UhtdFyhbnF^uIb5Qu*NDgz ziAV{kG_y@3Y<$_Y2nq&c8*40FA&{Mo%Yc(F#|&iZFp~>ATzWt>S!LU%(6`}|`OG>~ zk+5XpcCi|o1*s$(|7uUKz8jqq-CJg9y`cu@_ye)n6FUiCy5MQL*(&3*U=lN4)axd$Bk6&$&-F);n9>JOXo_^c4f;O5x;qH0fC)fejjPZ+q$X+ZZSDkJT}CO|C|-(p6%D1F9eutW$ZmGrZ*0 z_iA3Kz#oClD{R69Er*(?vl~cVniL*E7pYRQ>Z4+Qe8NgsQ_*l>+aj3NDnL8e;K8`% z5?-U)EiYx9w>P~jT{^u$R=J6nPasn7By@?oXq-pu!l9T!6(Gw~_F53X=@+Fod!M@3*Qsiz9Q(UA$Y&wjqgxO3AGl>t0rRZ^!uy-`{E>b9TR zq)>%W0CMFGa2v}e8{Dh8YYir2-pxSAG%1!`M!l(RJEtC zpa?=U`#%%N+o+Jc7B3X~MVYOG`jOPTaxEJzf+eFwkPdS2oPU2XL+1Mx1BsqskJsN2 zTnMrN#>)X>xEB8li#n1E%%RExB3}Ww#0^%f55Z<+X0*g4Np^sEmq zAbTM9c<2nV#&k_wzR|Y(&X)Qj{h^k7xm)|AoZY2S?r)DW`)BX&r8xJGd%3-ZpPk#M zB7NiKfJwGXfn2srfdmjNSAwnobMoV_$6*B^J?MR6GVX)Oyc=w8e`y{A3}NvV{@Khf z)Q`XD1+s)6UxfS09&p$jvShdu|7T z;Tv0Yo_RzUU>aK-8d;>SmoW~&xeauY6&iCFmyRm341MDNSrg=5J>?Z!J_lH_qXz#~ z-x`XhYpJzL)MFbvPQf`9gV7zclv0vVv}+|2ceYZ#W)(j+8(@EJK%|?gg>%wc>9L0W zyBF&YIr-?siqE|ZX4mlDf(s1d*5UKpMPL;4mKlEOrd~P{PAw0`6fK2pQ40@6af%=^ z5=q0&=#Sf&pyMnKdz$PhOFUW3u$*5X6$J~3ALrD%2gto`XKZq2mXf2NxM7#?iI+v- zxdjb7wpJ0c?}Vc)fylxM)96}|TccxH-3u*LRDvgn?!Gi@C}@Q}E^x<$#K_SDs` zE_%b0yziUrZ`qQ6B5#iT4RPCB8?sfjsON=(ZImoHaHC||W(#q{@MH*hcf(Fx?5+z6 zT6MA5eDt{3m5IZX2*#6Ti)ZLzX7ysZZ%4Vb0BXX>R^iQjn7<_3vL(N=c{=La(p5Y< z_mO8C>e;Tts^f;matrHS9^DhXdAb5{{Tuyjw1Ww?hZstsjKhSukQ{DM^If??bZzzw zr47t=^i?~)V@Iist(1DFdOblGM)`c&vgKuj#JEZF@{vT~e3(qB{O8s5yU{nQUZgEw znb`$&IL^N6BJvUOUq&2yzeT=UO$)(9>_%+SdJ~Z;tPXnuS(?g|3!Y)=xuYdw2GP2< zgQJwEs%6?JXY(d5r8G1Ldq#s;R#$Be7nQxUk!BSd`dC9zxfh!0@w)fHx;FT=p^SIb_IO*@R@ihKaio!bTl~?mhG>>(=22?p zFust6%1pl2sO3QDzrC@0{;M4t4cao-n!6TiFI+@>jTS3>E%}y-KxwHrApppoO}0t^ zL&Id|^rG9ohA3+k?-zAoh6p&Ez9R)S75EZW!ahO!*~0I8-&hHCrKW_Bi&)HT`2r@Y zT@f?X=6N($j$#qBm5W#h8&{E-`SN+RRqFy~OWzp&X>?^RXAu3wGj^G%bylZ_#sbL? zn|hx@R^Au;X*;&lqZ)2Iqj*B~UMTkZ(L=9Jm;2aG%$8f^1s>hl395Hg=kM;!@T=4Kdh&Rgs~zZ#p9;E1aZ~v!mf*)7kXKo58L9N%abqA1oG9 zK$1F3FPvHZJIZ!a0Nnf7Xy%l_4j@r>4wl`h1^%r6oGEnQ#POHkplpoFJn4@!8ZC|6 zxQTn;tBf1%(~o%SI33nDpIH;L?X;P>HglHli2-zR%cDXy((XuUi?IJK-Nsoa@I6rO zI*~HVT%`idQqQyo#3@@W{mbzcp3xC!4L-j1UqwX1fS|2*_H0u!*PO6IW~y`oq0YYW ziZfHhy-8vEg#@(bjnR)<+}Fp(T-yuVX?N>G7DeUR29yKwhxIxajT8-zTLQ1q6dcDOACCVaPTVkh!LY(kDSj?Ks4yXO4TVu3@X5 zHJ0#CrwCYwu#qibH+jV6+g&VYP{29!wX1^Y32@D3P`2_dziRv#;q#v}n82LoEb9wj za-hv5Zp+`=VhHxc?@c(mMFFb(0gzjF{1mu?KPz*uSa?QeH99D(28ws=77JdCUlUx6Ezx9|nV1-NJV+nvMJc4O|6!*{LfJN~^ zg~PDr;*ayg3n(}-aA{Stb_fXq?n2t3t?}^Rk&EF8 z9(%;UVT5DT%I^qvd<#a2Q49VYvgqM@07>~cn9LQ4JoA`|%#Fho5K9xRilGZq z!`A)#%SIRUrV$t5h!<0(vE5=gr-=xRiP&kxURW!m+kl(f9z!s5Q>9K}Ml9jhmL9t| zo6kTHdgOS>{&&=FxPVWQ{9y^b%}^i}v2<)bJ%($Zi3hU4BZh66>2O?R*ve+a5Oji% zMUL1Bd@SV8@QB$p*dD&(b|_`1cR}`e9(vo}kDT}CQfz5mDpA|RxD>Vv1cWerI4>foj!g#``Bv9-}-V_aJqJ~lFZ z?4-DuiLt}Ybf4<7n2E8`W8;R~=!r2Renv8U=w` z|4EfAv@{^ij*ZujNpbX%$MUKTV!pd|#iTflN^uyIW-nl}auvy7|IcJ5`3L{Y#LQLd z|Fa;~tNs7{hY=v`CiZ`4G$76Xw?$^P!F!%Z?&{#B@aGHKup?{}Sp_(Xu%m)=AI1VF zjvn3D`Q6HU=6tf)FqIi1C_AA`w^nE)S2X?adD_i5hu5&iB_9$!Pz^?uy;@cx0s;rbtDl_8#rge(Ea=0 zrFq*rXM*72>x60D{IB!-f2*$!SNu;{1KGSfLb)ZbiY)dlj)Pc;7oF#J$vd+(@N`K_ zZ}y_FUX^P4hJj~`rh?NO7dIC%87lWIbZM7{@#q2Sa$F!x%2*dr>i`*)2Rfa?9X>j*nDA>Fp+SxGho=rnR=@gDq5O58rApx`}wD{~cGm>Inr2#+q? zu{EBhh1I>4d%hyW4IYk1`2YMgUewRRXV%XX@?4+Wx+0YnXyiqnLS7yIJ9}2oR5mxJ zNcFIVd(XDUt;Rc8$`FKWE_lx-Loq*Bx)!9~TO+SC+U@Vl%4W!}x zM)_G{KF(sF+v5>Ev$T>fK~0>FnpPvsB+6dGkC=C#L3R<>^a_N^^y}#fM|>P~cHW9h z&$8yg!LDf%EPF!H%~D;k%1x|apZzguVUf=>%Qi0_ve~T*b=lJI)*6i8BFb(>HS-V* zW;rA4Z;|O&$)a^G@E?*WwE0rcxkD*nr@2~mzr$LBnQLq`!5W`<-guhWc!HUwQ(0I< zxiq?MT>w;ENT>Z>pnM*6=F5HDd*+0{Y7Bcx{&jd>98A%$LY-GyR}1^ZrmHquWF*}R zdmrxptsHQ*TJu)8rbvq1cZr<0uH|ZTpPAT|(u+Fpenf6A1-Rs}z(-MvxIu55c$43J z&KlaLiGV(jnV%$`s?i{t{x;LJGg+-cxMfwT@svEmuA^jwxe2EKL=?BA)fkfwRO^uJ zKVkq=K&-#^-5z$A$`PAEpj%0%7e*kWcI@A>jbdx`(Fr|~so=<_rqg!Tih609nMAB5 zLnDJ;0`5>&PRUzR47=cHL3=VKAr_ih$=D_6bS1?jxUGAw&>+0=Ahrgujh!Yj1+62! zab}%Wjl-1Eim6eco5cMP;m{`R<;h2tl_rQ7wn~RNaIz@Ax4P|nhV5K^K(vQh4hMhVw*1h#gxW>}vTUf=Mr!@?;#58ykw zlQaGu*J8pCpx2*wa=N*^-RaOk9TnBfL@|)E#n<$&>so!fg`gkLgWP$(Tz2TW4tbxM zQC5%8qWj1yC(13vX(-IER)54JMDTHc+<#pHt{&CtKV<8@){@*g%_t=m39FKcTD`-< zy!vV?_HDqVCPr`!n_Km-NVNw*z!G@0lb(uew@HeDRZc@1v)AK_m$PFq;gaPRob}EZ z{9Z7l1>&0}Xx5y)PnwX;pFsa^OQTDDPlBG=UAI^YI}M@uy)L7G@x4{Ql!5r$I}f;~ zO)sz%P35oJg~DB`l%9wDut{1&uSuiK(hyq{^_SVD0YD)e`-x`4lSBWgWGXonNL)Oc z>&PxNIqk^MMYE_ut7_~{C=(x(QL1?;aaWaz(PokOv<%YZ7PBhx#GM_U=FtCK#^tRGFhW&_oj>3*qsKwjtB1ZN9y#X=4Y)f=+0)IMuo>}3i z+tcapg>(wiTl_(;T)t_=&M5%iv4sO2CJci^ULLY91?8vIKKnq(vv#iC4ogT)ZXKSp_mBTo838aQn$=74>`(Y3;iN3Mg z-4|-H>veY&vOkT4BTd4^pcG8^lN;iv(4+e@sW)=O>7pAvuk1aUq*1A7{nI)@+L4V?r}42@+Rmi0;6 zMjRTh^9*7Pv2Yk@kZ`y^=PZwDq2C;Lw*opA8mBI9bQbe*L>HO{Imh=-5n4}$V{;U}Eny39H7xvDV~Bib7n z`pag$Fq`VsMGSrS(l3J?7Qe{zf92+F-=&e)K9>#-{C$28@qax%-llA)@#=QL>Flc~ zVTtw+)0uLSX7G$G2BFpBxa5)a7{Ka)x7*^ENYgG48^6qFm!hU9I$O?61)2*7V0sP_ zxN#&u&HjI6`crVG{}=QNXJSpviEZ1q?POw`cWmp9ZQHhO+vdc6>-qhkw|4E?N7q`b zKiyZ=_qfk{A=`$K4X;X+B5okmV1*}a8oo-;Xhj&~siyVJ z-O~am|G-aPy-#@I6`yvf{|cXBV3^yOx*~yhht|lDs6FsHl%ofpav+{4#U|)<|5HY5 zgd-#@kfn}f?kHe)ke_7$N*9nkOj;&jGL3HWcN5dlvs2vUk{To=_rS^)9e^^7NxcPV z4jey(x)dgEJ{l{(UBab46`10Tu1j^e@2J*{UE;BIn*YY!U39t~gfN_LG(y*b%UjT$ z5(8F0g$VOU#wIPWCD${RrrGX@;)0rhtO4J)TX(UMWLNW>xTdHXl6sahlOCV9#N-74 zsm?GO2B~AMR5&@fqR12V%A$3pXMO}TQLxXLB zZx?TGm&e7I+Me!q?|V4D? zIhfe$d^-ugan74sn#CE;{MoQ>e~WIZ_nxfYI_Cf>L=*6X-~%&`#NqSv$n5+B&9m4Z z_E8y2z~3lRAt8axibxzLCaJE%D(Wfx=f&C=wx(%s>cym}Y!t~j{EzzkoxTOllq8RZ zFnaFJTwquSa4qajOww?NJb1&b9RA9SLo5ay8uMzPB+; zL6UO9^OPIO-e`1eR1R*(B~U2x1VqbxyZHjgVo+PbL~Xm$@c!l^IyP|6bJqW1P%a$a zttnu@G?C}iBx!<3FiqwQC<&(Wk$_xtqnDJ|ys#30FzYXowUd2nf=yi)_%mR!KbSt~ zx~lj02)_B5=Mdga1b(?#AWftfiVkhAz|msaTrQl-=Ba zrKi3ET47 z)Q>hjMabDJ0%LHDePyt=^Bu8fdYLL9KjFhXZ;(=&tL_(Q@OtD_bs@3?G*Lt;!@QMT zkR}P-?-D#h?STtC2-O6F?PGOkdzoJ4HPg&vqZkn@7IP=gqIy*ZHw&q@MWvG3^uQ5``8S^ z4M|pos502*%m-K4fO*5;3#Z5PR@vYpc;3fSM}K5s*KO4ZSpRp56g{Nf#IzFrb-oh+IU_SGK5 zNWP08S^mirs1lCb;`lNDLFlx()Wej;gsFUwuDwCeF7o$Ed-Uz(F~_mZOUTKYm2P`< z>bj=aTHHXqL8cme84BYZn)!IzHWq?~up5Hd$9~YccJzZnDfMeOdV1%4>Kd6Ip!B3z z!5(ynFk=zO^3IS@Kzuqiz%AHDs+j=OCJg##^ZCq#ZbHN!e3_ve@qs~H1J=dbREIM+ot@Ay-8d}8ik4tGjG0mitZ=Ubg`QL3`IyT>IR%`}7NU zQkA?;=(*T^B0AnXeg#myYtbpIk}5&h(I9EcSSL<&29p8g!UUg&rd6`sSu1^}mKxCF z)`jV0zir=ibhV)C*Ze|<{L=`4*F;^L&+U!gv2mQD@oin7zqTGMC+$5M-|nlNaqiwm zF~;cxxLH!3C3}3GlmrSvokCk@KNL42`j}T+zmt1PIH^H0%s4o?$BDtJ(^Q>Bn<=r~ z+Yut8Vt_N*)j6Li-T^A})J-R&+pO<-Ds!iqru%rc857x|`-r9VGhjr{`!BBNZ@t3o zvnDHEE-(QxWovfV7noqqAK8|uDMZ*~UuANSO}oM~;(RG^PStH1Sgl)#Cn5N!LCHm6 zU5LqTVwOTS<4F&bQ41MEm-+!93341-q%!6&#-bA{cPwb_?X_S6G~itGBos7Q%Tl-- zx`+{s51>jBOgk%Wgh^6$DhvM~jHnRpl@1v*dkB@i9mlXPZ1It}c8Mn+WcEi@19*%ESU@lWyb zyq8t5zn6&|vwBo&-uQ3UDa83dhl-v5#CQKH>HjTi5;*@CjNz{^`#*11 z7;j7j(K>BWbU9=_w1DRNjmFzY@JjN12;Qb4epOEkC;mmd>5BxN!S=0A=4Kj zYa*6{4tUqT`Nk%GA8xubOyB&9urIcHiDuwOoUxM3)(Yp%3G?-GeXAM~VREKTYG9lF z3t0z{ggf+MT*6|jwOn0*Rb3Qr^w-bK8k6UOwfjeZTwa3(0X;;I^2Y_aDF};|2x_&Z zBD52FwMkCHf#iTX%e<|kACl!Ak1^R?CSglY>t`8ATBM4Mp-LCE&bnWz(mg))GtJD; zqi0wiKBpaYtd%ILl5fMLS3GL?>mLh?QokBerS=}z8i$R4ZzUr(Movt$oTta4BK52! zJyu~ar0XgrB=&xyH?E(_l~!zfD#Ejn_&L-4_h`p1gZOdeJ)ijG@%J|+G^O}~8*1H-CZ z#u14&gP*^Ts-tBdO6pyP7+pduo&COoX#T+kZpf^)%Xa<;5%^~HUh&QJGq)T|r{yyTSXGDQH2o3v zXHlS5@rx0&s8=s8&e+4;@N4WY3BN{gKFP$EC+=#}`NYBAZjqbJB}Z-+mvC=&I|w|k z^35o$aRlEKym7=-RF!2U(@DAq&r~9{=`re=Li-1k4Dk1=gQD$_($Dybe1PeVN7a5Omsk<5`%Qf>SoelBWNW^x zk3X4_nuD8!)E|Amv;aOJP%sYwG{tfLu>r45PlNXu@*|w!EPzBSU+I)~9f#mR8IYM@ zd97Kd`=us^Qi5bB$}H6}r&7S>IcBFiXtTrvnslm5>XKqRhhlEJ;}Lx$`~rA*qv{}^ zNMneBp5@f(^RIuy0;DF2F3K?nY5~JO%8!>rF$bQUwf{&~de*b@M9^LyZ2Kqo3tHX3 zj~w4V4vJX$I8tet(n0yrzh;z-ykDdsNn?9P2-kciU^32-N7FKvVWdc9@LBd4ZW+>s zTzwGnrJcq=FR-zpLp;HD%DD`29)OrqW2t4vBkR$_g(&;Z##TRtNe9nIQ|f~K(IO{7 zHuOjd!Q%J}XUB~RHw~j6LTC|yj$PH9*8XA-Mm?D^1d%vY2{I&nNUVm}LENS2)k%^L zyM-<&y!dj9sNV!qqCh(?fBd=oU4WKk#lLB35A*%!sHsXCCcN*owsyCN_oo(<;iGj6 zE{pc^%@BK_PsLw@_CA9!8dm(Lv+ldDFrj4h3oYB2qYNN~yde}~u=hKPZ7)Ia>)o-ovRKRu+>H>$mz@6-1EjsirkY{sQuxfMdA$G8obRcAOp}gCE`vp zWOc-F8F}c)aDh%$31r&)zopxd#BP+9AIItjR33k0S{%=`^J{Bwx81mfsY7*rlO#6~ z{(Br>HxT%lD1;J_MovRfg}izAqmA1MV7shDObFAGoWRiXQFzu-H-hU|-TtF^m4EPf zvtbnXyH>451e3@;Z976;Kh+&U2adir9+Kh={tL=8>>G@p$?w^k?tadH?IVZkg7>fQ zca1{3FPxrkpjgPC5GoBr0u7w98je9h64*fh-{}w z6PZCs(?ec9zko|Nu#lxvt+(WO-mUuO@r0bpf3P5-`yMVs;CN0#?=w^!*Y=2<`klX@ zA+GD`hxnhgYxSSr|698;>_MGJs>Cy2qeX|gcQR%qYg$M;RiU^dwlfGpth)?n=3>aQ zv4WgfyA)T7RP~=tkv==A(DL2U6_*czpi?5jBBxKVwz8{@2xrd^Hc)5nk{dRW*P*`D zCU$tMK7VY9kq`rFA^3e-1J7kbtiFfegESd!$NaO*Il)n&?C3-ajcWB^eG<-4hdAVAcI~HX?9Dz&V_m$qNgIJfsM)oA5S^Z~9BzGU( z7m9i|Lp5*`th&w)9s4ytUzp14Nos=r|^(=peV^keA&tJSTdd-1kI1&<>vQ{?S8 zHL`(H@vXdeav2OtFIBbcYMnWpFmD~RSoUN^@iO|b{j7sHuTDT_%|G}^;z8_mP76}q z`;uemE>hG&^+b0VqVnsbFfUbOTC9B;3on*s{rZd+7Y@g^bZ!sogRfsHNyRR5i=gID zf5kAvIjXH2X}W$LFH@IwlRnuMMz0hSC}c@(A=OniGaKP$nZJ0D@ZFp3I{d;|!!R5Q zVJTQ`zBzen{~_LB(~_&QtCnrE(0%oa=m4tML3h9ZP13ljnAhdemGgI3e#)G_$1&y# z7QPUeY0uI8LT1iQ6y ziN8)s3bs}Tl`f1B4OV?bp@}8$J|e{22NRg0xe`yfUa}F^Ek2gFLL#LMUBN!fsdA)Oi`qN>RGZ1b_S!=d z?qO#%@6A6}E$z<&itP9f9zaYK!7Kx+i(WHQ9rv=}M$dU@`*|B!<0zh1q0|8`p^&{M zGKoP&drKW5PtK)4^UL?51kzV*ryMGMLeN_ewxa}FD;|@%AKL=yzNX*-AGVG<&|Ddg zt(x@>-SspUqH_UM#Y{Q%D7KE3-=MnpSn~|s;ai76Z_k%yh6QHKdGd&e6Rhy4oqN47PzCtZ;y`PQm=K<9T`x*)DVG3BwML2()c^UO%K9&mH zPWoVx9Fh2Qmd$0MUu$y{=x9cOPpT)K;_>%Jtv>^Mi~5hnunMzSsuE$ltaWkuX z9eRp8DO+zHX4k@R=v433;mo;vXM6!gFL-_2&+vEsOQ00f#QdO}%-n3;A_wiJu47FP z)q|zzZB{Q%Ow(~F1RD(4HO!jZWBxw{gXD?AXu&y^WxuVRP4LjzOv<(PPxZS2nt&Fx zDtm^UOP>NZ#Wo}+GWK{ba~d(Tu~sXzV}eH*%j^hPiZK2C-vjAKOx)$J4=uGu&)=m# zyoXlUKKCLk-_3FlMWG{AeN0KeiL3icGvE^9lj=yzz*2MuYT*%f=|3QF2Usz7jQO4bce?Q@%|Uw#;gGxG(pPU;%EqJXn6qU$3C ztF?=uO9YP#uZ2`>rrBWnwRzl-0YYP42;vOwh4|ctV|+?{Rx^42^;ZQtXd+>T-e(bx z0BI9UyZyUZ;5X_jkk%?(Z-{48QcX5DSoUl}Vtq0nSJpQWTR(+Rn8LNb>&(rxG|nIK z?VSi3wKPQc(P)08nBZGz9|iul2WiaXvtR1iF-$XNUK_K2#JfwjtsxiATnpni-Qs3H_T#m zptP088i8mfd$$+QbZ%|rEb9>Q9Y3N9ZgRVl)>LN~3qyJm#!^B_l1wT!pz&6`_g=vz z3Av_01C=q3!-V7w-_L$c||phzApE00cg3-b>@hurda|L(Hn^ST+( zmt_WjokGC@5->U#6|n)g;oA;7!;-`Dss}HOhaH0*bUuI$kF6jz?P}%F%RxRSBY;_S zb^zPV>a0K6B&6qHKoV}ut2Fls?NsUC>M$}qcFh6TgE;k_l>fNFS|7P z==<#Zi*wQna2kTSCIaw|6n z*S2`o$JJ-Y8_$l1eI}%D7g=6Ibpq*1xYdhU$H_M=wyW5YJc0hGeESNOFVMlrW}*RQ z37kW(WMU5Y%tSdB6k}j19@;rdPAZ@4cZprF{zbP`#&t35=n7=&Wpj64%fBl~s2whI z@z!kvC#6CQGEoE>`ggr2eCde(aFjmmeY$AvXkG_(dlzUbFWU3Lj({WH$;T-}kmt}_ zW;O$7H9(QOyMl?jTW`8|uueGoI~9B2}F_m67!#{_f@6xYD!0 zm}i1|lB3W@xf z+n>Uq!%b@6Rm=F5IM$d2CQM+-Jhw?&Z2p=lsJA}^5?>8ElB*WzZe{?{t0W#LF3d%> zNX&VQ@ohq>4)oEE&>C-?DRn@G_GnwT>M{!b#yf!}$eR5%0p#P>l~gi(*~BWTg6=5a zdu=71%0cZ9%vB+HdIvlpl}yWsMAyQg%7n#VXf`}%Z*nx~mv zeLa9(ef`|ZI63f1lEy^!ka|xHE}NUeBtA#_j)w zRI{tE-#5i6bYoXux%`he|IdtFc@_O%7~ZUhqa4-sv?FFB^-$nXWw-__&583Y{yaQ# zPX~m0S%ZJGkECd}AbY}hGnAkgq83g9T>B9?FkUk#AJe{v%3bLFhx@|#3zotOlgCM3~Rpz@Um zkP~?e%ux?Ly#Vm$NDAY>6cbHLfe=rLUiCQiIoxXE z=8@I>5Sb%baj90r4vB59VhCHVn4ok}Kbwi4q3e0~#r(n4NRr|mTR@gcaY8W@8DNdK%+b`<0~d)q|cThgJVrn0hGT2PZ-Xb}n?= z$iZh#Y$#ok!Qg}dn|C%ziL~kZrct$g9ZrKZ5`nehNghJ7ON&=p!#P2NvU3jJU9%CQZ#} zf%3wEC{U35z_On}QAPNFZ(4`ogoRnI{i0T?1Fw{h3ch3GmVg6M)`Iw$Z0!BQo@UqX zcvfDLE zy;GD}>x%I7;JX&EbJjPg(Izyp6X!dD;Rxqh7@9uAXSVL-Vlr_1P5D@3d*XG#(-u#) z-C9^~svezKpMMLN6(CW0Nhu=wvZ@-Me6Kotw;&2Q%G`;uJjz2^Fk27`TLxQt6CPhM>%R?5}iZB zneRopiv?z7866LpC^Jk#B`RO4`jpT#q>RSEGBa)PdUY_WP*S%(0*t|(tck!#JA)9K zT5P5F9F_aPAXG~=ZsZpY@81<_-NlQ18MBp~q_6Z+AvOah8kar?L+jzWU%n~`L-qj>_O|0Vq55mdJaAyYacte9b~-^~4$!J@pP46kUbz6mZu zYkD^RIKFP*ubt09xANX$Qhl;@Rct_?dh6fQ&5YBUQjywWl6yzv14F7-Uenh_5FlOD zJfVGc?x+EI9ljq5=4Hy{wRg@jx@B{6aktuL_qXr)3qf3w;9vhQZe|7ijK7s2Nc>Li z*LMb8lVC92+Wk&kEAD{8{?`-YvJuu}l^@+tOsxa900+hlL*(dUWn8_?`h2Z?BtvUz zP95YDq`trA*yXFGUsq?N4>WhkVFx|37L|B@M0)$x@03;qQQ&ELIb?3ebC+Pe^D$B{ zMrxF0#prpmJ7ii!=bi+gMB-u8WEb{j!s}@xr%mn}?wAV#Nt^rec&3XVojaz?0!gMr zQrlcs43p$Nt*He`q6W?{Af_E~n4P0(C(6e1SlvAU4dhL03*8Tu!eE`%xv{w8W;(Gaa=u0m$ zF{fXZ6MRWtCvI@26To<`ikm~0f`bnJZ>e#tlGxdrLJtV}kso^vk}!8QGvlLmJwV0H zLB$AE$?rgv9F^#LllQc@l;cE#KbweY=l2|wyNhJ0XyX)O*PZ+7Oq@<@g4K1I)yA}j z-CoA%S%5y?(NC9GYDpv4MWQc-xou=*1=C;yxYJ+HseLgk4I{2ZuDw^Kw`TjaUjstw z3R-nXOy=qWmABk)&!COZa#4JGnO)g!)pX)I%Z5RO0Q0M9P-*TmR*o}GtF|{fzb*g1 z!}hFD_;2NEiy0Oy??#wHmzuV|(|puWdcoWjl%9>(6uKwkv%S)M+^x*HZ^QrqC1Sv* zn)e|H#Lel=AKjV7JC>gCZkU9KzV>O?jhCe$bOO!gncj}!wL+^$c{t@PJy_<_TTswT zNWy>PcHWXEOd9!36 zMdhaA$*B_)1@Q3;%o!MvFrsQafxi1$dv^NsU4PGCPDj+D;hK*WQ#Ra19=bYR`;U=mbmOGK6zDME3V7=RID;PXu#LPH z!+v6IWEI_43iZ^k9itP&wmSyCAui8L?{edhpQj%<6rUDONNGsZG1K|zRDzw~Nstqq z+(6$_6cmLPYrlTs0Ut*2OelPBYJ{4%+8Vk}mQr z$FblITrtsm#(%-xUet9#W&j=>pPSABAnV|Mi#m(>EHMG|&4RrStIHE^N-V9jnjZZpH+Bt_@1-x`d$7+ z2;XFIH~@As5bxOEuUUkdaO&r)t2g%+3h=GyYd>}VMJBVM#!b{Vs(C15LNEAAFr&n4 zylB8ROPQh<*7MsnHSFnB#{IX$d3EaH9O5pQr^Pr+=FgB=CMVX^FxjjARmLBKN3sH$IW5jxXKUw#?8qSnJCLh zD-=K`J>(=SkXbq-@mlk(QOdvTNJ-qH5UzBWH{|{IsPcL%`r=~bxiSB1`z0A;g-R*S z3@@w-$Bsz|YLjY&;tZED1kcYyu4AhN)rHA#dbyYh{-()H+&}d79~VuCE4JL4=p6R4 z6gMPgXMGX6WSN=y9E**M^jfUhJDrI?ZP|?gaEvVVR)16&g-HvImCWsppDnK@W+ilw zBnoam2|QJ%o>TiIaG)rC!1b#k0P{)t>*M@ZJOYiCB-{Ur|7lbj1e^EXE_-+O`T@~m8z^Y`@x zCZii|7fTSbU8FU#Y)4&8typOv_>*BLG?B0gwnz-B!zd^@gIwM2$b&SpndMZpZe

    z;{M)l!~{fM)}1)4(4a9*?(192)kuNC=L zc%B8LIo+Q6vkmIAYRCVM<>upfMz|pgvo;LnF`XF`(rTX)sw+lehyg8b=vosyTu=TD zU5Y8lA8!89X9{Ca<>*kE0LJP$8VwKzaZ&SnDSmNt4n1RA3fzBMy8%;#* zhOoU}k!MM=^Zn+{0(h@pnQ%3oCX4F*XUrAhec0S(D(MB7@0&S;g2Qea*@q)kZ;L zm`_2Vd7Ou$%{$U+8~&o;N$P&Iw+<5ajScZD>R&WYq(4aYgmKE&=#uB9h70#PnCJcb z$pt~GJ-z-ec)Jk8IY?^wKI9iqd8$lsyAZ=X|9>6h7cqg<@B_%N?(2P&`>njxNA^^# zk>BqWg(GB;jzRj7y#n+?Yuu3{ZSjpYgvV~(CE-JuaS45g5IR5M5GX@|(N$87Bo*AZ zOzcZNmF?S&KBjkAXBM@)I-TzVF%D2f4lsOPUiY8Fmzh!a^!mEj5X?wQ_hnRCp?BAP z4%9U{a0ShPblZsn_idcTM4fqr&UBoRVuxNwH3*)F&;}=XG{Wa4@bDXr=(YM=N?A}7 zkx26}Nf^awbph)@GC`+26sutw|1wnm=_0HEVR#XQ9ge`4(C+jV7xs)aZu88Wjp;D< zQhx24N4Vt{qgB6;SDxVD|Zc&`~C*}AoT?V zkzqC;Zuo_Z?O)ptyvi?#m=Yz@L*;@eeiXzse>x>Rl%>mYEP^8YbNDZt`7IZ^G zXf0FCYoB5zzstq2#D@^5nc8qrmf6<$2H)lkhr?wfYZ~C?-j~){jGvz(DdC0E-wCr= ze_;5NP+fGsQBc^&auE=J4nHt|US|gf4eX4cJ)Za-0ExVXLPJR^e|gC=_=e_!F{|)h zb4a`X!s<6$+Lp@DFAtl(&1RKirYgBu&$^2Rwrnzn>N|$x#TNNA20M|-G%$;C^vvV* zW!jgmmsKq&vy9bNvL%`=?*%=f&K7s>Kr$D%UtS@ofFKF+_%>eBkm@AmTxzG!fMy%$ z=QXFVOJLm~)v(6t55J6L>gxV*Af6}{BnRYop$%VwN1-{cEi3@*>kUDvx}y{&-toU^2Ej2 zDb?rQ=wWS~pX<--;{E;nyuxnp`}4`aVTQ-o)m+W`4wj$yr!$Iw2dR?8qoW4)6yq0G zh}6f{+LUl|jcl`%$%((wooS2LNo<~o3dRy=iz}gSg;20xetPV#Fk}@a3dHpRA5MF$ zn;%^#0=%G8&3MH9x;QaV{m(oc_^bs;#G~xm2>9>mY0zDxGomtU8|Iofk6_$tz4|>R zi#iLpj&?k7c!NpB3Z;Q~Sm!vfQI`LoZm{0k;KnIDEln9aY|t;sxQ38v4vYWW3%0o| z*1oW2& z;V~=wHVt__>a=@dYmtq?#y~GQ`_HK&8Em)y_+bKJqxa(g&3^$ES=FkA)Xy}^R{E3g zP4WQzF$^ZGW`iVri_?)uaA_oJ{nzuGgTH|ot9;c+)RM)0YvtLbm$EQ>*fMImkjg+( z4+&f*Jvqe`VW%UbTCZ8ZdX*}@Kv=Q|JZtd$Kp0EdWld+_7`D~5`v~?`C{~4xA7j|} zkMXTf+hI$+cYkeQVNPR0N9Bau0$=TD?e3$}ro8rMeRXq|iTR-~9$G}jKsZDljIxty z_pT4(4TNC480cRq>8y04nFtxRBeHM(t=`@b5alz;1_FKKP+8e$FszmGNmm9fyC&yt zW}aA;Wt1S(vC!HlUW}_0ba4|x9qO|`bz7sOvK<+44Nh#wANP%favP7oU+WXZ`o)3a zFjTLpWL zt~V0s27yBx08h2Sl-SDaA#1}z=_?%q^SgELbG?ES;uvanLC8|aIIO7S`b68(7 z;~$GQw7su8NHtBSF&bbl&~@V@%avh=JJ{-Utrf$RQB{MIP^*(FdR8LU!-w58Hj{n8 zCj=J@{&MYa(oHrSsa?qe?K3oo6&LHs*y!c>InV+9)MfQnORcd+7M1zpYaH*{vW}L* zjk4h0NRHjlNMUO)l2h2Y9m&!o$~sKCKJbc**sjZ)K4}*jhih zgT=x%Fb9slJ}PUqwvA~r5@MJ^np4zhv>;%!4%k9^ZHgjWTzZ?8mo>jJ6kLBT=a`J9 zXH7v$rzcKULcAfa3$np?(7Msrs?}#?ovmgM8UK?DyXg3&Ddeg|vyl=nd}175GCk}z z<~FudrE4`Y&RCNRU~9X;T9mX9OwT=$2TK<_NbO~M@eW?srIt=oTLC? zlxArmRV1RtbA_)-Ho_<9#B=pOi!L=oot72D(^x-CBP?a$m#}e3S`)t15Erb|Yb`_C zYPj;mTQ;oZ_&xkiXcFi5QdO*%*U_7Gf%|?!^n3k>2xn7LEDF)y<>7oEcX_KX=%hMP zwZqc_JCK>!CbIet6ZbYJC4R#=VQ&SWmEzu!uI{NNaOWv+O6|-XDBtk21AdYRI|BOp z za{a*$&&@5aI+?Yr`7vh@<0@xez)@AD_wOLVjKjTYqTfo)y5|TYt{4|3xTvfyt1up4 zotfFqd{rVb(o!U_hRiwGG>hv^-2^+kSHJ9WA2BDGAs$!1mM zOL<`HJ5vfZl*n-GODO}Z-8aIS{k)~YXtfz0e}wM^jGytqn~~xi{8q$kE1@f(TN$K@Y-bRr#9iOO|cEMq`tO^EMgz-7j@>i>{}( zy!d9Egd9I&+13K9L6G$jpQC2s0S#sWKHD$}=SNpD7IA6yd!aMI-r|@tX8^@mLUp^M zn>JD({wX3ZsV*0E(H*(R>Mtb5cD!^O_b%+P^Xv+AG+-`6u8`tOFnY6 z`01%gGq3p)y_2jW$9t?}68Vpm|z<# z8cbf~G8Gum>%!r+d^FC<0z5}j4MZi_$Y-i5nX|)uziuuBQz2}v>-xjL(bFcPeoxx%VE-(q|C2tc0d!lYM9JFao{5G2=wLX#naHS3Q`50w z<*?1NqSLQL+4|xO2=YTu!P>!T0U}QS?lQuBmwf;AZUI2mOS#)42mO;h3Z(jHt?DLc z6c58cK6)=va&wH74J!`NNN1-96X#R5d@3Tee5tuU?}5Od3(PfM~=_ zidchdJod0O{#w~Z;-tnRG}Bi1;CKVGSUpVCB{`k2qQLKE1@qN#0?Oz>`NXJ>C359! zh9zRMjHagopJBdm%{Z$)Heo)8-}+uY79q8I_S-l9WR{0XDt4m=>ln9@DQe~e({Ok{ zKbkKQeF(g?coF?@^qw^A*w65uyr1c9SF!8u4g43w)5&F|+0{j!%9vdY`&ubq^p-GY zw3w0DV~qoqVmx${RishayEz5mw=?rz%3Y`fKr~yCwAQuyOj#X8A)HW5<#4l72ecs4 zsDQjFalGhN8!_VKjHwfd*|^UEPFZV`d~aG$f}_p3JkD^#Nub0Wh#07tbl zXBf#~OdykD*`d{6W-;Nx^8{#4lr!CIcYs`I(WhlFi!jv-gsWr2qV(zNX@1S#zeB2} z!*EgqV11gB*#lx%GYni!kd&t&bX1V&;1h?~B5noU)5d=J2#`ZfHY5sp zPSzLXbNDvUSBlKBBHBE+fbnHGT2dN$$#nzAhSIxC@S8i;w)}^pz7#HfjosJAgOw zKsYIMcR1NvTSG+ISVee>ZH%(#?g0k=9E-wse1~x)e04{(C9Mk#tjy@ox1kMNA#?xd z^6Mw2bJZe8Lwm>tJN=q>qYx@Yv4^%R0I4ceGDfMR*3?cM1c9V3{b0QMB67uMnx(;X zIRlumvL0%49&6H2k-q@w5=E=m`_crih+>@l3re9Xm$X zRON&n3iheeDh|P6u|rg@5$Qb(pg>aLrPlz$Tm?gr${Plg%GY7k{!puZjo8G}Kl^;t zRG*L{M`m}IhCryb!WjT{g@eIHGHgAD&_C{?Xl`DvrnZi)yqCNtJeMdFHd!`;b|_7I z`VRFtXzkY%zR83gj~+RMW@aE^Xyk9>X|>7hEn!<#v7p``mK>r-`io$J*8W@A{cztm_$mR?EJ1fR)W7r4Va*Feju0_S8a6zbStc}hx?iM`%X zUFC*M#Jvs*HlgA11n=^V$8VllWxCox>*i6?T86n98&_;pKgn<7J>$SQ}k&{pj`oAQCEa{S&5|K*b4QSw(xB2{SlQVHp0MFn1tIcX zy_gqtCg8fU68@|!WJYe%C&)2d){KI;9+HyYq~SPv+PEpxl#1)4`hV^GyI_Q`twNa{|Kb0*>r{An5XjB`x^wxq_xssi4}M?em{P1VBVZYES{$2ouPo7IJdn2VV*wAVq z)zTvAU~qL`K6Mmxv8d~)@6b=bt0=sSLuux-9VC9d3R3E6sStO+vETve7g|n@-%-X3yTZ+DL%mhRn!mQ)7xjze6gMVngXg3q4Vx90_?jRHBtZK~yjr6=e4pG;j z%pycGXi@Tv%te^{V1(IK|9=25K+nHLT_bovpOUy+J`5^Zy0WKM2#OWObf`noTP>p*pAEj+waOiFPXTxu4!9?v* zx&lX!Pwc*Bi)yNF+7q44OPRL^_!B!c@D=-5-^)}6$JbTGZ+XGi?3K!B0TM*ItLS2c z0?bM!3#AV=Ei{D%b?c4=yV9C+fzTzTC9KfrtL3+nR(b}k|29Y$xZhIB!lZParKYu|G56g#mm>1Z^#>2Gh1rYNgJ87?}lU|eL2U#LaGJWtsq8U1 zx}pN&LBm$N?tGd$jEl&0gq`KA4o^qO2bZcw=hO1EFY=t_o$-mej1EYOd11+W4oGtK z_WJVwp7Lg57UjbkB-1ku^vawxuI~(X!Et2opCs*0mh`~fca7*GxV1FMR_1I$u9AYV zx=;E7cEyN(T+|#?7Ej)Ygvs1YucVwo_<-Jx72jpysL^R;Y5dQ`4^ zg5*rd+DMN2ZGh5L$&8>CDl&F9oub2=6~;qZmd}+P&@21a$N|Z-ji#)IS$=Dm(r>O9 z&7;1wlGYp4!M0T0upGZI3Z8QVf|75W_&&D%tj7<;9*h?UKcd!a17}i-aCE=b67}=b z{H-W<@1(51a(EiaE9ZNZNPBKr5R)j-o`Y>iZ>sR{wNM@|13?ef_;&l*1~9hWszzL> zuPfOEPoO;k>+8#xdYxzq6ci#+S35w?*@MW8`BqU-{#Ngh-kxaVK*FMde7me8l$8a%;bPYb$vSJy_naH9tpj#4QVQ~6#dDwtM zSXOVO&>J&(VO`otQ&;@o+mzrWHyTd}=#hX;70$s%fh}eQ>J-k_-6#H*yA@hX!h~unGf}>c$u#lX%a=itPsy2X<2kG?sK;JQw9oW)y_F>&MHPh2 zLS%)EXGVl1s>wEf0eNc}G-*|_V5{D~OWa#Ql6`B>guNBp_I=WOd+TAi<+qAelbGeV z56momkxsAiF89I3x@9%ZX&v4v$Y5Qe{|VC_RYgbGY-qZJG~;Y`gq23r@EDu@ z-*C}<#N?D|W{~Sw@F>ab2*J!^Bx-K08HvF-mz}jT8xd!LzfcS_V-Y@GQIX5-UL&<2 z%sL?1iw+}7S$anS;PH$=1-oSt*v~#vxPC&n!cH}iAO&5r!o);kaCF>6Flot7i5`>1 zv@HC0vS65xVmqZtd{%ve8>jS1tBKt{ffD^W^{I)Ry?cGqhjq74aAJNweM&M?DVAk& zTV7RCtD2L}0P|?;KttS!iNVLfJG+EvBC{_F>0p+%{OI1blaMPF398#SlJ zbs2^yCwuU^udgqj@?04FPH4X6O2xu1^mWIU8!2zdS+F$DT=BJfVix;c_aF|8 z%|+pYnn2SPwM)%`MrN)TFcDZoBF-&#jzV`Bux^B&>IXY%_NOYITN|COH%3A2;p|2` zJ~(D5+ddq)C<Q+vn+6y}sQq29~ZGS#r zkW1lGL&9v9pzYHqWP`41%?bVbIWRu7P;tCx!%MYPS~ZEQFlKNjzR`K|b_3(Hc~GAt z_K+PkMo0U%SNv|!Agyod8l_Vdk*?7?W|&T)IAj96hXs&4t`(3BSpt7H14Hx{KX5oP zsMilQ=KGGlE1B;_Adhy!;5<4g0Z*(G(kN9P;G(Jjkw*gijr{|Ip1Un+!GK!4CN;AS z56Hi^wv3Cv{!d^ZF38j({469L9|hRKvLbi1+UgUJW*hLnQbF>j zf|?ICqxo*GYFcZdoME=o_DLI#$^^R@n(%$0|F=Dm&&nR@pgL*%`m*SY;G+;ladQq zaurhmtRcv&w_mxHGOqbs43psahGH0@O>f1xYj9+chY~Z)SZ9S)VaNs?h-^cJU{?9! zHH4cj852a7%sA92QCzGGMrd8rY%|U(zJ~yL9N;!m)O@4riqS0&bj7xm<0}ZwDj5iI z-8MzdOZRf37zC-~8R-TNB2n6~(i#2|nljTpDf+mgGl06@mh}!J;|vsLrD~EDoCWOj z*h0k8gZDf1V_|x%OqWFtl6YAw5`-UU8-TASIafwIQZqtp%lu&C?YP6#kH?+hL4~cs zTm9E~=!tEBp4)8WTdSaJw-IKY2<}$)GBq)1uDq={N)i@|V7Gd0vy5r6fOdDJd*Q(w z&pRVuGo|R7T?H)dW6CQS+uO;2ebIC<@^;&shnf&r&1FvDE* zvv5zlF<#`gCEgO%XN1TDRMB*^tTC{kje0q$nfhC%b6>qVE$VT2F-rK+mi)RRyvTO>p z8h@o%HjG%K*nXbQ;C>2?s3qOwJ|>pVHc^^bbYm^lJZzj5S$*~^S`x@|A0=IWb9+=QU=+h!GVZM?QNRC#DXG3VGEB0)af?KJzyTXj?rrrV1ukpJFs zCTte49@a0O{{D}+O^B?>AD==zs+u7~0-mZ5%q=Yntz|`xthI!RA6nQ4uj@K(2`7t` z@8AE2mK(>pVy#%Qnqh{7)6`Dk5IQy74N_9}!RZb@Uv6d5Y}re8%TtTbb)K+RmVzrl zL0T3vyCJ#xu@2(Aw8hgNxT>Y!!cOSNmX;+KYmAq#=p8mUsNVq%z1|^r`N67MBH_?c zKG3iBAv^3Pe8#}+Nf=b5&ERl-#4|>Z5$wLG^MA_6c8UC6KkHx+3F#1|pG`D!ts(^C1N7gzm-@dsVZ%=kCXCj*Ge7{2teh zbl5yUzzF;;3ts;T&iPLSm|LS8I0rcu6%av5?@+iTC#v&@lbkC>pvG@zy z&|3!SV7#DJG?%KD^?WaQyluaLR`jE-|RJn{3-#KcOWz6p#rHIi?7Q|~$;DC(%hf|wF^E*0g z!TB1WDFFW)kj+x-iUzyVMZSbLwc}{K=ryYZwE&nU<(M1senW*+@0wy*w!Cn2S+TO< z88y%USi^JwsPu#XFu(oS0&}6~he9b9G8kuRDS>&W^?kbloBULvHU){X?OMt!sdc^q zie|OYf>rf^IWsVC1t@9&7q&n2-LY-&bd{-@8AJkuVTchaNFimpq}k2T22ye##e2y` zPUxC)p<29x%YoS4&}m=D^;hW4jPyjVjoTC4gWfu5qYCXlgj!~MB@}FuTdMdEk3%UMg7z}o=dx-e@bM!7}%VvL9&cV$+@bw+7_WL|X3+0rt z*mCiX(R@5WdNAlz?0NoI()#@=5$vwmkqon~M@$i3r91t47(v<8h&H%_I%zG>&MK-l z5Q8egBRUuv(v4zu_1(Kd;Zt%}%G|H7x%a>jqftaj31CK;O%H88Z^KRmp+&i|fhQo$ zwLu5u+2V_#hye8QJzIS7doyx-!78wWuWMSfRa0EE`gM1*255GoRQ7%k$0&p%6NJp! zH7nqJ|2YnzmykSa#^sH)!A(~($AB@FZCKtE;4YrTr}4|{T2^$;Uce5HtogV5PqbTK z8G=ON@A?EmPBC~C7I1=E5YPe2U-5#WkGha|5Hy;D!un};Ch&VN_3mC#Y%7 zLAm&FvKS$UMfdaha+gH3tmIa6m`2SJ3QGoPYkh=@ETy@9Obvf@Eg5`Gw_6WLShqc^ zi(Vg$8rzmK9(-rJjK(Km|GDrv?k&yX10YoZBPnL9UVUV5TcD`h81yI5eDqm@YWJ3w zz!^v5bxcry#guHS%s~5Y>COUBE_bbop_Z~RQG5DKACE4nSY2^;%LuKOysl`qb7+hZ zsIh}RtGCoY8&>VWiV68=Arw=Df;YPoQWmx{=c^8*>I)mgvPPp=HD5P8XHUFI42b}v zr_Px5hdl25zosSVd2qj;Zse9Q=O%!#i}W}0T3M~B;6E|L0~TOwO6~lu<)w}mq#Upjwj6}&c#8~I7z~_Qd%SUKN~MD1BB9& zYJPR;if4}QJH3EG8`m>vx1X93T4RYb{%3n++ptCJZDZ zIN1cDrRHlDr?ug{w5jEm);yy{F^ZmMp_R*uZ)wGgor{FTe0J!G6olt&TgqBXL%67X zactCxy@G^mDTYkIllL?S80Qsdl|n_~0O3BcUP@Wj!{E8*EP77v*fK2nAbyt~QrMp^ zF&H#yk(0HSGa%c`A$kQIFtsE<-m&G6)5K4szP>1^Qe*w`n{4hUnSOv!rTAL3;?|WM zFnt7HZv_>7)~i<^v!cma{#7Nnuf$4LLqY!Cy?7O+K;|wati5?i*3-}1O0|Xz+KU(v zvVk2r)D3al+%W&49rL}cZWOUjWTe9KH}*-e<$AMFW^f8 zx=ynhTvT4hmN5ijO6&inAAKcq09dy{Jywl3>s>RFa^#HAG=7BlFR-yck{91x5?ZYr z^zpyd{K}c!tp^X9efxCr?AhY!pc<5u4PS4D{ESCK=+O}R-^~#E&(1E5@Q&V=#_lTX zch;mhs|aiNd?klsjZM6kG~C4=SilSH&Y#cmM@T_~wb}Z%_cz1nL7yNs53y6D2gb(Z zjR!X7x!xVS)SW^5=R^9@oRSr3-8-j@*@?mpZcHa;$!#Ia2U2q(9duw2ufQH@;&Ci0 z0N8Mi=BA!Nl1iI&HB_e}%w4mB^0Bz)t`!y{*cFJ%EqJcc-kn<7R!;p)0 zW@=(q%uBQAl!-)}>}amez87~8J^&j1awq7PXLgP*3R{G4gmxD?z_g={l;zuPQ=_Z; z5{)S!%?C?{m_y%I4I9SR4ThT!E;krVKgv%EQ)gw2mZ8 zblO?Ud@-;_0`GA}Vy$^P-X^YgQ_AO80G>-Sf${?5-o{UDm{4gU4% z><9Mh?}Z&X#_J;2?_WG;#$+YMR_2eK^tvw-+Ywed@YX%E{Z?&0Fw7u0f$}O!#W%(wEMUFaGL~eOq z8yl&|rpJPpdELQHH+urIkSRFK+cG9To_;K?qn26{;oGtR`3`f~txN=NmqpuZ<1S_& zGsbe$gt24gbJ|>iA1hWfF`R4VtlnSze&24{Rpp@~t~HVPo?f+~0CxguVh|3nl2r&& zhAVRV#qT$#7?#zOKPt9v3R>x@t`G)={Af1IR((>Rwx2ei@=tf4cIQ71kUrb;qTn`! z(*C>UqNyQCtf>^q=}wV5#%>hZF-nzTxaW7?=iK|e@jh?8&&vN^ zo&WiKe}4zKQ6-z2T~%zwKStD?_j(BR-K2!uJ2O?WHCIp=3{bK+ROH2Q6@e9I<#T1> z_KiyS?WHl%8dt6qgc8*(=OrDA*9~>aH~ozVLJr$Am1{+TFm#&1fF}XbV{VzrneYq$ z4%t4mTHt`>TT?Psd+`h=DiKgb#`E{WB08+{S8idaD0uxx5R=V%MhfGLz|rd{v12^z z{idhvY7Ot=&C9WTjdYdgBG4Rcw);i%^HEXV@IeE{r)Gqb8@8L7kRoD3W30+hm~w(_ z&@^PP*28`u?P}EOIWQbIZ0E2IU|jSUE@$2i1s1iAAza>5u_12MlAG^_xndBx0ZT>q z;Z&Kr`%E?-Kf-pCs6zW3l@ z(~^hAL~0lVht}u3cn;gm%Xsx-H^f|#OF?91$G~OYC_~-{kOah)a30;I3Agm<(!W?&P{`2fE_1V0pD%57;!sz-#@Y`te??Ot0}tszV_gu0egzhzPx)KEc+ zxE)}e)y&NkUwgtHux1Z|Ef5K#l`NTC{ zE$#oa;kUFfaWPa7h64NuQMuYXEobCzBNh6foaO1{hV4#fogFf6!aTVYCl2DSyD8l{ z1yaNb{C2X~J7C8Y*zmD?sH{KD-=T%DMA>WabZ@%cUWICkDTWW;Xb;Xxp#2n7t$mDk0#d&767O#|HD!qE74ng7y;PU-OOclXa>g!52 zWlFHZ1zPQCK~z&(YkWc0`gs#>fM++X&e2L09=JWa76gGx(3S(Xzf5Z{sTGl4rr#Hfy3A8U(`ea_`b1||>T>}F zZIYiJB*@Qzcm=5z0hc`#I2F0P8;kddSA~c<8qMx_0ShdIYTR|v4Ls)JmTbvr@X+ZS z#t-MQp|-&pX3dyldP}AkD*IRlu+>e>#g*Q2UeFE9GjMIagLitfboOEhh)eSVS=Dx^ z$dW3Sw@p9mt&($y8i5yX=%rL|Kj2rD;i%PLxQ8)Y!PpPR_inF#~l4hP{9 zBPWBXmh;NW-R=Zqyb!;Qo^fz}-0+6UFwBo!k>`m`q^Jk5C=k3I3Wt zC|tB^#6>d4*PKN&zFO+%Fa%xH)ea&e_wZ(2X`qoleXE#qH6ZK(=8C&OZW{m}o4#|= zKVKJmv;)kQ8wl?Hv8<&iD(@UXF7xh`V;?1ybhMe8wqFURiQ$2Dius147u zH;hJ?q|e~kTm_D?#2FlT>)6Up;Af9?kUbJVQ|ucqnvYlnCB?L?m(=wucnC{Ie5{Sy zfO<`C^cp|Vcw8#*V;4pWA(@I!u0B9TMLw8hps#+*T*jd{KgUD#eurq> zmj^}wuH4k2RRE3OJ$*T3o=Yi@kQN{6{b!wYO_aIXQAg5p#ytR$+U z@39Ca+-JNO0h*dKx1gRr-C6?7aa+=yLZ_I>!@h=+)|;8}mm%mMI4V)xKUF@4s9h(f zxdEa!-KcF1oMbE9_gs;aZ|oSKn0wy5eIM?+P3i~yO-z^El>q?khhvQ{ zZtn#Fir)%YP`a&it&i-<|!P+T|Fy1m~2vn*b5g60d{%i%dTyi zNTUwv5gu3ay3*wgX50AE$UtQngs0h&uP;J>Llw_` z8~`(Sm@W;^$2GxmF}NeS3+Xu!-I!L)0C9sKZZ}0-4fS!XH-Wx~`iSTxwdMh6P35<2 zM%XPAgpYN;_94Q!Fn|;0D2BE5v#NnXFLz4S=Xfen*uAWFDyzp_%Uy3E-U(hnFC@u@ zrRSEG10D)YD@Zqh?h78AMi?`@N9C+9FQMO~E(!-;(Wy#n8KE|C8&==RDpa}^M#Hy- zTfD!>stFf-b1te0+W=fu<`K<@?b$Twx(f7YT$Op*3lHnMu);>8uybq3qmUAG)?SC5 zY>`;uLr%9xvVsPFGytL(Stc7X3S8O=DC|El#4Nja4Yb%;RK4|OP^~Z*79fyD`Y|{P z2H!u{cvA(ya%g4X#y0ZKRS3Xj^HIZ1SS-l7qRRqHz+Q)nE_Q>ld@xQ^tdKNN1>K5Z z72C>NXU=NsF+@x)B^Qh;7L2_aXCM%JG+I%v+v->5UM$RcZ4knx!<9D5pH~bgAisLj zD$y-*DA6=N$kFY6!ylrbNzh~RBpfwwSxy5TDl!93fPfW?-C%>^#S;ONRmln4y z=R>unhQ}7~_7geoK)T(P3=((DFBW3S@hdoN0tD{^vTYxOZILPw(A zyT+J$>XrV(_IM_Dfh#H=;eTdGoD*4**K7+dyx*Kfri`RXVT%960HkAl>Vi-)sbDdQ za54Rb;+kjKa4lrqwNy_Z|Qdayu!^Jt3h9cZO@+oi&sD2s3#2E?IrY7#=UzcAP?+#U0k5irEYT z)odYA0b6gN_q62O(amFiuTtiFTTbx8-*au4S`ld}S5SxrjIuw&Hu>x@iVj36ikNxa z@sf1<*_-cw{JENRSb^JQy$EviM<_)13|8XT7p64#9~R^*S(##Ew}{cI#_TK*Daged zZL~=6*}uUVI}DWgH507jS+93v*6j{T;(^Q1jr!c3wQFp|eD9}258nYnA=Srl3Nr(?T{3X?_z55o z;OFSs>q1b}K#(CanA@w%$5z|5fezSN&a^~XQyF6-z1ef}pCE)ReG zXiQ)~_P)h~EkG37;OgxEXfZEw_|%Uk_X`wQ@L0sdU5T}RRt!LSYCBO*ztz&Zfb1A+ zOBAw3!U38HQ>Mr141(i6(rsC=8L6ctg%s;^z#6>}gni@+_AQQSpw6_hdqOKLikWUo zKaU9e^IIY7-`eFIX{dd$mW19)ZVQ1!IU&s9&Ru@Tj%>NoQhP|xzP8NB4W~kL$+D2y z4a-e=O)N8J-*ENai}2bvsu3EfA>l$R4s)Blqz8~IIACj!J%`q)8?%{P)*FgZKv;Fe z)I}0zi#hPsu+VIKtr?*E114H6$QxO0X;ByqTqCxJ4PQ$}1k0FGwA#&}06cfanGBjY z7DA!|N~i+}f+++Q7RD)AR`P}kk`;}rJtW>XBeP;OpJP4_*wajgTSyGDylG2wSTTQV z%dOfnA>X|qbX7B`7^{bT$wu9J1dXm(1m=SPd_66{&d{u8EdPQRfw+kj4;*oZr*I~7 zx9D!0YdTi14)$t+nMnz6 zS!GIP{a`(NVnpl-;|Lmu@sG8o6JQLq;G=3L>hIdKJ={b*u$ZWM zJJ1n|7%#!hvm4nUMVtC<(-X1PvmToAmB>5X`&>UVje>SfB*ee0d z0Ix(y1OMg_uz|fa2SQPB>fE$XwvJI>qBT~~gsw5J8O(F?+m480#8g2`wb6nH;uxMO z^TonU!s6zSFhkwjL#{}B{dP|xIvd;@=P3vtcsY3-3O0kC%GNP#3(>(qnW&bOiQoA4 zesEPF@+zoXz+j#7AlO|3MJrulzN#4X$#&nEKor|#%HC=U7h;El_|jV_o@SRolN&8k zAHUvmb8T&lx0gIranQe7I`7yDgYPE7?R5{}Xi>4gZC3>xTrC%x9<$arX< zT#+D3X#cABTGZyb`=jXHGh*-euk?d}fbEV4l5qQ1H^B4i5(6lWNfP4;?a7^l!QWn( zGwwue9A5to-Q4Cxs@~HBHuKt-&$u{B| zQF`RcmM@?6gzjXjnpx{9_W0~f@7Oz56#C!$aa*+*>NnQ(gn9(lke7f8iPu?P$+9%2 z`0aVW-?oxIK%7}j*Te}k6~zHZjmo}dW>^OAJXBew!eY_PS2^U z2a&QB&i=w`dQQGoII!+qaGur?Z99RquaDbW)TnST$k|0vZoKz;nr8}SXrf&8#-r2b z7nQ^+;6U_Yn}5_>k9zA-Z|z?HmGxFnvN^DePAcje*}TpkxJA80Ly%+XBt}Zm7U*d~ z6!_g8-7N&>0&uamR`7TBtzTC(7*g+FZHXWkt!H+T zY4r)zOYUD?n~h)Y&xJkvQJt}xl${2TI1r(JAk^=|>3;RGq(bZRUr}C+N7eh6T^6m= z1F~6E$EQBYuQpsH+LcGfP``RZg_g{3xolEKz{J2dHA)DWWWQvl96yrFXh}0(?@WD+ zR0SQRkeb$wYQ;y8J-`Y7fKFOB;}0z*hAPtbuPNy4d(VbL!@E~%p!o1p$>%TmJR^F$ z*Fo{U{cCas+|+Y;pib^DRuQ*2J2s))$0l|VPP!*9{gv+svV{Y8sQ;D zNR)bMDW06OnW5G8w6(jt}6`qu&HcUq;i{*_ueZBplZ zRn|r-sZAmosk%0)vReN-Xt3jz*ZNm-CALWe)2-SzTA6JU%{pGiZ5)(uoMb{(w})xw zUX`~=WWyEVQmel0QJuo`P5%yAms5hN?Uim$!z)xpZvWm$sj~hxQE{8v-)I%O{&gQ+ zZ}JRIty-6?+jVN&MXJ~(5{=Zl8?0)VNII?dovU(}tfVB@!b_w$wIpr-dMu6Nej$g( z68WN_=oXl2!R|jl0F9F{I@cbR*3o03G#}c;n(5c1B*0AA(n$Nll@vU9Kt~CAd5$UK z6v9u!0Dd<_(#zQA%8YRW*)DDkL`qy=lDy)#kSyIMe8|eAj!yGjxO{7`I^!M`Z&fi> z%G5y1!R{N0VFQS7Hf*aVWmA<>rL~!gz1;)G8k1{eQ|mChGi@KhG(du$P4m3y!pRsH zM~j`2;(V4PE1-u%Zo=b|n~CYSr>7X#;dz0yI^n}ormfaaHz8pd0au_7Sr_gzQB@sH znnW;>rEG**!+8`1-Cp65)cv-()b7Q%Av}rBu(oMfSj5U^N*j6K^O>z!Ri(0l%S6Frcyk;yZluon$Re`yeC4NOV1?+>v|2z__! zGf`4V?sbx+8&_KAFdL&kC!$@C-jIx@)0vi+W>H%N0}H~~!%MoV7u!4~S|x!KghN}q^J?Q+ zX;&h4lVk(J-@UqiADmPp3; z#IT_Uq?QZvBIX{|d;;Nj0Eqbo-Lm2ZRl}eV4>_$60X^5#a=(d< z?$GS)L==TF@qob!r**4SEAs|`>5h}0xE65k9%LEBQz5$ZgvLSR4r>JeXT#bOC@#D` z*5OM}h3X=0xifx7Xi;xuv)*8GacXe#mYS3`LhX47Rw6hGG3^L5=vwiLmYOS9JZ?V( zLvu%(UAU|eEPlp|9SBt-H_=ttvuG4P&)K$=H7HN8Wc?5$6e>9uID8Qiw(aE(^Qo(g zL!pfoXO;R5%xKC&i){FW)fk2@>jS#CLbgS~$~Ls3Syh6Vux^58gklsTYzP1>LNg2xmIVG4c|*sLnh5+AZS+2%RRr`aLU9jp?12EkrhnIS(vOiGJtZ-ELdx?5J9t{E^TaggDJi}a+M1TQJTYnc zQQDdo`IV4FnUFx49KUjC?8*eFtjB=KdK8GPsqk3IgdYVO>k(kF9s?3?%S% z847D3=qe_2L}0LnK(1m#5rV)P1-OdtgdqIY2)I>B=VQ=Uy^w>Lk`af!+5@$Ui4zgX zs}Tt)CT|7cu13JDVti!;>M9M$Dnfi8<|+-uD#l$roTDiKREXbs4vq>s>RCrU>!@d? zwlxv%>2aW*Ccr#RMru!ocuG&~NrHFk0yia4jX*mMfHox%4Z%7M0yZVk9D#ItR1m`d z(sqSJK&D3mV|pYYrima-sr(`tfT@L7O3^dZz?ZtvN=Z`NFAThtE(N6mE*+Y5m<%p6 z9av^EsLW>#D06Qxna2QC`fLG}CgI+t0x2EW_^)e?KNMKfB%mYezvS3wKQT97}5biNO5!UbP%NUgMT{s(Ny3gHw+1jBo2C% zn#G>l5`m!8{A?m*?ndS%t(J-t~ zGSwmYp+QKYWQzBK9-0U{bO7YgH29qDwe&p+hyFNwYH*O2~i& zB{U-uMra=(D5Z;02%$bWP?{+@1RpdA43t227&_=);?zS+(}xFBO#=a%3;@)D_erxx z{=(otiJR|0(4PYUf8q+wWU!w&s^3JQp9BZW3N=I#WH<{J?}+*3#LT_&Hq2QG)kJ)4J`~=%#%!a;kVO* zAU+%VpnU3c&!*Hu0!+6B(m;M6o0&t@-p*`(k8w)gfm^YR@mr>*L+c^mFj6WApKvgC zdt`h{$x&A5SgPA&+Trpj%z@KCn)nZl|4k!lnyICtg4HHE(1$VDP>~l}+={7~hv7b) zjLtYhd8o8tjiFdO#eXs>_z>R7Z<)|rYC-N&-ZWRGC$J5Qku*Cb{GLZ%OEqT$XT6#f5XyI%W zJItl%S_$LkWU3D*>?76m7Ma{W+3w~Uyw-XAKAc48UX--52}VnK>kK7xL;!ICLXB`O zh}z;H=cL0QP4j`0b;YrqH~^P8z~=@|TCF0R%u%i4F31NlxVMvlN=#t6T$y#|&?{jT zyl`9a9W3NaaaGB6#Z;890el-s-yqfCKJYo4s%Zt_P>TO#j+!(@nmd4#d09}w76a}4 zit<7q*3gOsBZA*Xtv#>cdHt&WhHG9L+g_w*)s_odV5Wg?;v1PWY@E?Vp7+Kmu8;N_ z0uAMYs|~IqH;(QUk4CMnnhIF0d)k+PiC`>u3F$=0T#t2xV%tq`t@Kxe-WHnA;~Gfm zc(-TW@$5|Yq~X+5yVGr3W-y_8(-@)ozM?__)C=6MwOL^jm-?;6^fSY@Vta|N2>V!4 zl(;DfWcobWYI)2>7tVdjG^4TpFPWG9HMsZ7+iZcS-eTl*{J|PRPyZfy+y5dTPR4o3 zhm)V-bvW*i@~XY9{m5^5(NrE0( zNhw0plcPQea^bT@Q<}-h&3dyWVY0)Hw_(4Sn#}Tz&T< zX8(fDTJSWmAsYez76eu=5>zRfaTKp*e}Wr^|FYa2@C8_aqC|&Qr+6K z7GY`fzK*MPMMYE4ka)Sb%O626O$tJt7Jxb_`1G?4JiS-YX+-H%q^@XoqdI2ws4i!z z7(4fOFO2TCHtY=D@7uP=R33tf~rPy5?dA2RwJ0`q+8hULl(;^k_H zrY~DfrcDbi)tfiKc@7;W87;B~qX+T5Q^(fJVnPT@t}tJU<+G7qX!#>T)le{yCkB^6 z$hKAOoOYzd=;7s1RLshs6B>@T|2t=}unI_%=S z{JyqCJnJSnXp)SsIl8tW9?Z3^7hrOV7L)6mYEkGkJg!MT&CX~I2D!A2v7XgBoI3?YqpsK#Ah&xt-^=K6&R&rFI~);&WTv2N1AgVQC^8KKlF-fmp9!fA+Uc~OhGT7X1zA_gnN z0s2{t-1jSB&ORPr>|}G;Unv0J1OEy`7}z#T(0%;hNO-icuA+ z12oced<-Mv5hWih-n9iJlDbXx{Z&09TbgaSFy{!G2lNn{h)58-)(Jwd82}Kcp@D!< zZ`b6h+8fT)YTUdw+NdHY9fo{js(Khxw0{yN5gtE5>>2UkQyvRZI*=^fJarv~kDvFu zGI9Ipb$Hv_jP%hHCcQ|cJz)q^Gj!%@F&`+fCPk$rr#-~tlZ>P^7?e3Nv@b@gC!9t4 z=_f!8ha=mFGB8m@^N$I7o*Dtx5Ht~heWXtviJSQE*ks61;P9}Hmtb(%*8qn{!0L7h z>ma(G_hXVLO#SQqfaac(krwoX@Vs*6MPc})TN93lo10)F+n@7cs)h|~Rd8F@aYj5z ziN8nO^Fw9Ur=A1-6eXBY?oXuMnZxsxz-7IeymVLDWzZ*!N#kig_{AX>M+|8ZbArj&@q&0)L0 zfY+=TMH8L((Y+?+u(b`bw%`rbFM<%N|HPaLsvC|c{W2xhu$ybOAnzeiKo$nK8RBVX zPIx%9IIv(WS0)v~tL^Yc*6cZX!|u#JVTrMG1kGk`0@td6jjuPDYPNof%ni$K5-g$w z@>)N6@(~9!KFA!Di9=4ouO{+CKRG=Vz?h~PAxKhf5CCd{=K8bo9h=&xJNs;m5 zmnb)oaIhwdjOB)^De@lN-#_QANWVv7C|$mH^g!6CYnp9LY9jsuJ9~J!))Y&rLLEwk@&^-7#mj<`Ke?>* z8C$JbR{uMS(rteF*c7 zcplnidaaMxd{htUGSB`~{X?o`6_*Hw8nP|-Hp&@Rt+8uXt}Neio7#iB3lQ}~U(p|( zpL?9#?brqR%g$zG2V-OXFb~s4vu(^2wM<|oD@Zdu%OwdF&t|tgtIrqY|6!GcWhdC0 z*1QcavOcTYl&YtSJSAuF*myg)Tb859d2W+kF|l=QfmlBP7xlmXKDdZNE@5bL(#rYH zmThQ946Td3rC^TIAkQ3tcEX+L!6tdtmt!o`;VAy6UVoHSYz=!1ch7-4EI~#ZG9cEn zl!aXHuFHziT&pN`1svdDfeGmm1q30!%uqa7fEsC{SF&m&?-1`9-Wim3N|`G{x6(*P z7&2~KeTduxv5(gg{Kig)zVV-MVOg~#E2ySmS`RHe@;%XiWi$d>nYlhX965@ZLWT802Ia zNDJ{8r8^A(GTnBW1p4?maF0`f9w!(|(!m`c3)XQ0prZjKOdxtNl;eb`7KAZ@VggKK4}LLaAB+Mt9tyme47oVnFP$*>$(1)E(29wmiiu{pl-4Ig zC{7Q=OdRLO08D%wsKn_Yi7~*Q$?%A2A$1P{jF_;(5^k9egV;{b-`FlqM?ZPpdBzGa zwGkoP6B{fXcvuG&b&xvi&%6zes2a+SSJ;sO?ig&7Z9^Q?qjNO{wgc1|I(O}~+g zOm2@gKsUWRgqPw< z=3R|=eBMW)*2tOtuyl5*^U=J5$kCo~(8LeI4yV!soWiO;Uf;00Bp!A88gK4Z@yOY_ zVlXziyo@puTQtL|+HT32Hhpf^BKA2&0tx~2rXtlZObB@BRQ;n+4iww?U3ZQDDxZQC|G z$&PK?wrzEcQ>Uip4W{aQhIP3x=MlGP?_Y9tv7M#ib=S?lY`yzD>TEjRn_7rn@lZBP z9P6JrAP5V^g&l?pv?%N*HDsGTOap64A;BG(F^@U0nI3+SCPxu=6?$`j$sx(K7b>&W zkCx0bPzQZ3!=4Z*AaRT%PT_o8hF~OU@NzKKk7K9|`Mh5AY>3Z!i=tfc%&%3!$g&w}Ske78LZ_p`ZGU5D8JA^GPar+^&UzEm+1%(9oXi4+%M=Ah zB^N)xk9+UAtG>rWOlwSutKAxJPq>C^Lx^Vl6ZjMy-kqJt9HOOJG|w)2kaM^ACuEKk;<~uA3^fGha2rb0{8hjqC5|Y zqX8k0T3vF`nrB3_o4fg@MyK3Y?QVCVhL{<@27L~>?A8`v+;|f|;3DFI4b>O-w(ws9KDQ}1o>JF< z^oJ%+YoHKNv%k2l;1sgUJZuZ5wJ@k{Tr@|ku3mOWex$6Q0&5dc9b7BBP* z!6UZ=@uOPA08)o@29DE=sdpPP^IPK@~ItLV48LgxN^XVb2p1+q?qVb zzC2QOO0yGB)9inQZ4$0rk%W<6K62@d$fv^-{c2|2~gmzNg$XYLH&3@pz7 z{CxU2&~}))1B_eeQMv*JDjlO_gc=&n72*&^)8Li%dC)oLEDvP=GPJ56gS3>bwKyJ% z1dYEkKn_~A4xrfh4)i5B5by)3rYO6NTmF%%^L`gM{+&q1QZw@|c`mU9E3?=?kwVo8 z!!Zxlk;Uy^j9EDrS@~cf8Hx8M?4eH0wQ@Iwy~97McSrI4vt})%%@RRsYR8)REAgo+#*f7Ho>drzF?Qo^eT`05g~B3tHH~g2%|P_}!*L`Z ze<`0pi(%v~ph>jg`{P6t)!YllmvV9MVF2WBi-y)c?<{d->A(g?#0kjL@9?LSX~ znOE)4e*K8iSWWgLlW(xg3(3Rbz0X%P-QYc=O%oF?uf8G?T1EOlIvm%g6SPP|Km`q` ztLgMx`l^D14tc9|?-4i&On*TkXKfZZ$hg~cp<@s3d^h%RH>TG)m~NcPZ1r{?dFJt| z&oFTz|%7#zVu)eJqJs6!xsPcp65nOd7>tN$8<`!`3i%DhvKbSS%hb8 z#~FcUwl`R1hoV0^7@_~~%gNT+G3gv4yd{P%2c$h`3vu)DUW3fQfcH>K=<5B~+)xfP zD`VquduM1vM|dbWNO73N1xe?q_5z-9+(#sbaYSKggS;#aJZJspSjV6vgsCHuVif4r z^>y7_A|KBBj-3D$jBxYrx|HGd?HpOtEn-ui$qrRJgpqf7+rDTeYOizzrKpMKXLf% zP=far`377q*j&{{9CNa~Ci^gZr9BbxSmUgWVRM9#NbYR-vRdxg(q{~~O=7He2d7|; zuK(AZ9o~&!$aMzdc<`jw8K5duUqn+|$`ZlN&2QWfeTKj#9_Y(VYui9cusq$`eD1un{^{BL_j+`NJ1unzhQ+ zq=Hs^k%NAzwD%v8bxM}f$#%A#5-1+|HfD5Wb5H^P-+jK9@X1h7+=-N17@!%MT%=~d zl5r!a;Va(9Xvi)nX}m11extV@?`bffE}xJ0J##|AtB(jAm**R2N-QcJ#5#w+BVDt> zvj&FM>EIfrro13~n~u3Yhi$mwj(Rr$HmbvdPKUZ?Y)3;=;0~ZO#Bg9AM>zMPGV{3a zXT?cOKj^O8N#Xfl?WS9+y+zE2J}(c6h3DB{(zMin?*mH~o$r^XS?QZFk*nET88m)> z)($rtV4mNZC~Jft7?>>8uSM{#IbI;1g(&U4C9i|ZxiOJkmvjW#{?n;QW?ji2 z1{JX4dwSq1tBSxW{VbjAjWA%WD%RM)l;P$(0f?+kbGzQsKZ!3;^t_ zS`_-ZuWwQHK$UqcZc?{dy@@yW}tPW?yAQx{usOdwq)EKkPW7KtI@+M0BaCVWT#@>XPH zgGE(U2JBX*CxN)fJp%u(t+z{Ew|84ZLQyO~Bx!ddd|UZB9ZXR>@wMAm!2ou}2tsei zM?SOMD$80%)q1pD%y<=nwn$4?5AG2zZZN621O27p=X)YH*H@3`JE2nZ(oC4K&3u5( zz$QzqMoay`TATGrD3Q|eJ$Sr?K6?LiA+vboSdY};zpj69DmuBH?(Y9s2um|3?6fBs zUQx=8QbPC7lF`rx;&2z?#&_}D64uQx{T5IKf{_4zi1hPQm`R!8ENr-B=Xqz7I~e30 z+Z$~*NY@M^QXJ>97NeTfK=qAuc%;ax(ff56y>dwZO@HhV1*B>eX%QU>Br^K;aMWx z9q%cZ9t0?RXP4F&}Ou!&%f-0YANU3AOg6<@I=P_iRhmHWZq1FXT5Rwtj z?MxhH44=Z`Mw5Fo+xHfFeQ`chpoPTAACS#ESh<%0<)ZWaN7*m>Rst^1p|(55J2wsF zPr-*x!RmiIAr>58jx$xd`3;eSMtY739(qK*nU987CZv@1CW}$K0w!~!(Er(+a{Df~ z7*#l7;ynH7VD5&oyy8I$OQCFp*=Jw+uqdeC5enV)i``#cG|G`4NM}wUv5gT=l3z2Nd^Mwn(OjuRqiq2iI*r3$Ycg=w9Af2^EauU zuX~$ZgbjT5RFWr(jkK3(i8Z-V;*SF_0mTd}ho=dJ-MAb#unPSt9(9Zcz5Xrhr+pja zU0w`laV9PX5elOa%nf3m)fh0fto!3+;HTx@B@y?)IDFDQTK2OL!s$-tvYMeyS#fy; z)E0BnXt1%)TAZBs^l~b}K8yuCh5z*9#xa!L5UB;8S%m!a#d%JeVJDD2O?L#A2yd=t zUs-NYu>cGtP>A(k8jXuKO_#M48Wzkjig+Vcki8!+xZCT(&~bia!7E`A;V*=`h;cRV zn_m7y2+$PFo6Gpq9AHz`sL)UmcqF(1?+x+MxdxFKZbXV%^87K*A1F;c`G!8sVzEdL>Lx_j0<(S$?=|mmlW0E}< zi=jSUV5RS~+{U>HJgK*UGH99_?9Qu@Kij(B2Ya{6f-yY3eg>yEZvg`FN43g~w!|h0 zfYhe^cIomCEz=~e=&9G;CkcP3i@T=f&y?929|Vi+SdVss#m5sBk={<>yZC66@~ZL% zULj6w;CcE=jdjByD^2L%qUTn#!_M*z`FT3$RAm2uj|2P!z3v-Ea6J<}cHkq9;NQtz z57;kTCXNR-I_28Q*E{^${eE%8emK)QlW2uLk`W-n)FKD!G(Kzh_f`rzcSVd-o0ksh z-R=lf*GOg@q*>W$ger;Ch0+5>x5k(Zn|$q&^0d_q^C943ejm>lst|NvM88J?DB7qp z&gl;xMhsQ!>!u@)^7$K&e1X zm1ubwz)rvtG}rD}>yPF7#cJ-P12SLQ10Hvdy!24;dqhLPK4qfcJQ;4CBzNXRwlkk$ zZ7}ipZo7L{?#6wukhvEjIV7c0t~2S_Q0{~Gz6)+0xS~b3%A95QqJnj|$sd94PKt)O z9ONzst*U~hUOWZvJ)81SRxSo~?ZmZknxoTVS&Eo)i9?#01GkKdHkAT~IPmGs9Vk7% zAln#+5ILUFNBP%3oiN_rBC0&OQL7G1A!-)US$qP9E@L4%|0M(dLnvoq2E2ggJkJlT zT8?OkuEDtDi1lDxBlhtc%kGt^X-#4%Hjz^+%o?Qbp+v(IBElCRT6M2xFJ7<^!+JZCQVo|n~PHLeKsMn@Wd!j@N8}2q2MMEws zS7-rUM9a&eWGbI=U3;nGS`vA>tkWSibFL;Nev2Yl-X{NEv+Qdfb|Zqf28@Ir5@(z* ziBBo$iTq>aM{IbwaKM4(@q=XJ82zW{u=x^zt@yCqSnPkOtNbyS7>gBlNoJ_kN|e69 zA+2O==O+u|p5ZP=>#h)of@y0t0f2|5?<{OQR(rG( z{1M+FHB&}DCJ&EZF?bL1LMaC!UKnF_IO`py$8Y(Y+IIVZ0Axi_0U2LY7^h%E5~Wv162_j z%!R8YD>fJYzFaw0=@+ZY7nCX`U|JJ-eT{-CgaKVuzphU+ z#1&iHm=6!KMa+_+AuPo)GJ^yPT4~5(JmY4C=fJfmBejv4P}~bQ(KWKOI#i|x0ir=> z+KaJ}PHmf0gf+0IC(l8PnsoW)+x$qd)=7tL7y|xfV|Ea-Hs^=Ymd3RJ7{Fi^kkpyu zYWO5?TOj8A_9a`*XOBJ;Wm^Ar6S~Kx+_^_fXZyu)ZGXWJXlfWd&socDO?8 zg@ulZ+Kr?tdH>&V98Qv;FD{vtPuuNKR@1QiP6XissX6v&SVC`Lk_d-1#6>hNteNU| zM7|62h*#>qz%zLa`I&9>=X?rN^14_u#9|UF+W1e^PQ-4JmpNb5XYFvP@cqpe^DGRK z@$k2J%ye~I_5*qO^-9%|03W|Ty#mX9i8@ZUH^(oTF=@(xv-ZuuKbu;A@rJ8bF-~|D z+gf{4wv*R;-_SDq?->*pG@;?dfz-z+333sHQYMXZW;E0)_YBbYAT^j1YQS+^|BmrV z$}o$}t~Oh`>YeGNg#?fDY~Il+i#BYGCF`o_sLpl%;^`9!#`Ni3gB@wB9djdrf6=bo z^)03KD2E-a2ilbO6AVDL?lH5@%(N%nTC+vx1**eayFn8QEpjuklq-Rh?Y99m*jA{( z0S)n40=llv9)xi~7ne8dT@y2Ve9!;omH4k0VN^nlH)Ayq(zi>0l`OFd^m*d<;vbdx zC?1+`)btMwR9?8j7!p1nTuw!REftOC&7|$jQFruv zCO(i@pO7EnPiKQ2su-s!=UrPW5Uc+0pU7=cDIjQH3k#;F?s}BX{*;w_h)y#8EabSn zLH^A|)I*2^Zrldg0tR9b3af>8Hv@^S>eyEd<8o-vX8K$TrWc~gV2L7;G!*E*V1q+* zEV`@2h{D8k6U)2UHO1EkU{fkhCph0~-%LXZw_Yl6icFsd3Tuw8OA|<@#6Oi*<6RIZm4o@Eb*(L=kr73tNabS?ZR#0Nj_5^5l# z&EMp$aPCFLtCscOTkJ_D(b0+^5U&Ve=4CGmF$OEDzHfcCVhuv4GV%-p9xqSPCI+ih z6&EUFgUetwdL@*!8Vyj|K=yT?@5Y^&mG^8CwA;Zb@K&7w@H6%)zq)EDnM-;k)-EJt zPIv^5s(P^0DmYnn3(g~nOr(VvNCYg!1V1wHsKzE~x=SB1tr-k-;hq7RcCXgu z1R3}o()h1`I62`~>C_yF=LyQai7H!9jHA}4`i(?#~_4ubGBXgY{z zj23$cuv$T(|96T5D%3`UaT_p7huW|b_orUoAt7mxDEr%GOv7B*&6j?x=?<*gYu}H< zB0Jhw1itCUzSoZ*i)LTZnhxWtcEHd5O+rFR_P-JcE)d-rt$b)HHX|7?j7V&UAE&3A zE4IN3+shvgsl_CxwZL@v*oIdcqWS}4n4li7#PTdE$=5N#te_mOU%q$MK==DIqM10= zEND_`e z1;;RlaIUIm-Sah(OT$m#T`-LBVPdF&)GbO40z8UQJy8}|*JKliwo};!e@?P4nPX@X ze0y@RP@I2yijJiKfuGs#^In7%Ojzm_kFpzpaiq}%oeC+=?nfZpItD6jQKBJ-iGfDK zJ?;H`A9dAWooplSB)ma3iK7^W{#r^n$AcIY(*~a-bQSDs4VB(hX2Cj+dKK#l2b!%e zq5?y#7ZWoQU4vyfsE$pGMh1}9Z82CunYp!ZF9k!+NDHDlDqb&g9^TA6El>LzeOfR1 zbIK9OViT^5_#YP<=UP2I8(E@f!W#168oM~t)4g_!>Uazg8(I2ZwI;KiXN8@H^^@V> zxHjh@Bqe=hXw=lkb9Nv-PiYziI@A?{3q(umb82@=QoaVlIFvH({c^oT)-N8bYjLY3;atv`P z+=3}nL*9{c*CL!n=VC<1e!nrq9H&eM*idZ9(l4&IxGQ`5LWu7h9fS0+zdOk{Td)>n zZ)0Ge%*5Ow8RP{|6zR-c6BvD4ypc@dx4yHuEmen}zz(jT-G4^OWU5DsiW*i+;k{|{ zsIJe(cgeZQu>AMyXs@Machv4}yb#8db`6J%{v7k&O}4*buzu~Fh2(7c@0)5Dajm=B z=L1dz$AvEnd6PfG3MJAA4|?9Q1Nj4-+$E#=qV2g1R-lP|*wW8Yt8jHREOBk&)zx%P zKz9}Cx+#V{Re^HY3c0=>SZ0r&TJa|!_o)6NL)?;!opBhC>WOuo-SR%FxRZH^l{TTU zX;ANc;PmO+ZqLWbg%(`qZev@=daz*gxulZQ!y!h%pqJrLm1rDKd|l>6xJN2yc6E3t z+`XzD1*&JEx*kp8PKuQDKNZveh?B7n2M+UZ9ZOU*EcQR90IqD2_62#_L!W?E|43Sss_TAn%IRrKPPLxmzJg(l*FA*zY1hK zW^&@bt2oPzs%qKp78-zXD&_Ajs6vnZPYwANZ_WFT0+OyVZw15|xNDAqoT(HUoKQ%J06HTwp!tB`> zA>vbrL`TW5zRu#)d33>;G}9=dQf+7#A$sE{_qBJl%6+Iw1=%7O#G!A`C$Cm-6XP=Z zwfjD<<8x9^9S@T+I=c|{x4?uWs9WI1@X4&&;*gSXX$I!yW3Fr@(-fO6yHkb2msu&Z zYwfs$OC@pxnP@P?jCN|t3uF3;rbt6|*f zmz-!?r>mYZjD(N$)RV2*`}nxn3*03e-()~%NJym0)5vxAa1~u@qo|aWcCt-+Hf((! zHiBoXr?z&}%{1i#pAAix1khDE5rPGsDa;iWrCG?(gsc;A% z9+!&ZE87=Z;5iU2TQf#PfVP;Ztj)XV8M{r-Wh!o2TavKeHO|v2ofUrw^%Wy%JB#Nk zp@w=oAcFfP^1{dH*_VX5tANfTyLp16!i!>xRYQr@=iAHU83js(pBZvMi1ES#CZEI> zo>EJkgImB3Qc-})wWJzYQvW0tx&@LqZ7+?^<5qv24wi{NgX9>&<=j!oyzKHA7Z&*X z8Z$SwYeE!>9y}i&_yYR8t%W|2nrnb@!usZXJaXujqNr~-C$oM>9bHHFcbQkxJqLFf zkz}=hKm5HkO3>=iZ^&nH_;Zs<7#-PjeOC0}qV`rh4uba?xZJL^9XO^?V>$T@lniGx zJ6!iuQf)=Bm2Sf1JH&3+oxk=%66vvVleMMw812vNgzleP`y8=mImCYCxxi<;F zXxmgzBf61I0B$xXb92o)c!t|H-#UA^PDyA{^U6ww??IBjekE(@u9{ajXDrEx+FF&q z4Mj*`ICy9K_a>3(;jmfk`W00s{nMBrO0Ut_PqI=vzYG~6gh z6Y?J$HYE_DCI4e^ZJZr|n4R!jl^8Vy_ZE6i$Qc3j1Y4qTDWx$H>!|bA`&_~fIe$C_ z$f31L^GlP@g2rhHn?_=RwFWRM30_}A$`(P9GVHKpM!#XV6Gxs{T@I9MX9b!JU$mt< z@A--`$=yjLGy0t2tb!#~i$$xMVraB;MXt-!1Gi3+B9tW=QMO3`wO3e7m7LozAIcGq zi1QNo?>HZL6c;iKWuRMWB#Fg_q;n5|B_EUhXw|(u9maw*#B8-w(rD)l6i0`~z~H2B zu+M;C9iM5QcpFI3zcyIk&c0zm^xx2x?tiuAS6T3CGFGRfSx(MC>cyVz*g-tj)wxgA z$B0Ks+p%$y*p;^Xae>4okNR9}mYrDU{d@48&?v?I#h*U@@yh+S!aM~}Ye)W4auJhI z_8c>Z*?}dd;qLCi4c3~n&VOcL*?ka&;qzgEt)~IR=d5prv%hx@6rit@l{C46<$dF!#EITg1(0@1pu*eQ?7NK{Q*eohJ@ ze#RvK&yQfnJ%VE=b-R9u+O5XYs(m&9?*I&GK7iu9p<6}OSr`L5COC{eSDWcgk6n9- ztc2=Y?Fm|kdD^0M(6o2rJ^TxIjsfL(B@#lQ?QVl#SCIGWvCJ}hu6CrU{Kp~&+6x!S z1TTMJk8}zfJ?HNVX3&z@mg%MZXLz9w@6AR#DAcq%4`VjF1Qs5XGjf-81j znf#SqO>`ozoxFxTd_D|g!}PH&8IIMx1YL`G+it?PRV~6Yah$-5{wCWlm^tE2SH4jh zLtc9$^dj!wtpC?qNoVxoEqgW8N=+^eP1MzgibObA8?3Yieru_;rdowI=5v{!%_%qK zGkTPfAEPP#8#D+*C8w1@{GnLCWahuj!u!;J3{yW*k;S(n(#m#ET3qYua`b{c1MT=W zstG%PCDIm+5i-+nf`tD2;%;oQi6(*Z2+X*>OKvt{H4gFCW7CH3y3e3){pHXhlAGE#6M zBmfuHn%IU#bFq;)DL&A2CUWQOAP(&I5%`Bp&|3e9X@&Ac*W~OD@hCs^Ar4IyYZl|r ztP@FgeEuPfEah92|1Z4$SO>WlTdmCla$cv5lMHUWeY9%=^`um^gkui-;ny<|JHbn2(ee91e=6A-|XVsqiL zAZejksBrknPt2XwHJnHEQ@gj&eX#c#E`_B^U*ND?n5VXgXS%)Q*@0Mc`2UG5SL5Q- zTYGE8Hkh5yq9^QZ;Emk&=Y)29V32^rnvHkVuwoxx6pEm7&(?gkz1Y`;X@q3B6I4C=OM0Yw(srnn+lpNe` z^+KaCzU$)P)GffT_7o+GVY7o16yB98UJLVCNcZ2{gIukKMB=~a)6i(+Rm`&g2xM@^ z1x<}XNNy!rQ+fYDy~_At=?IQlWil2K8oz9{e>t*dDG}5UnHa0LK#)5aMx$po@M~A; zpMxQ$`YH@N?wfRZ0BsE{3a$wI?-vY>5j|*$35j5Hg-Ka|`Ywej#ugAfR^we!D2X4Z z$BvgUX5|IFEqt{o>`oeWdb6@-9z{m{M5RknbpRmJeN@#JVR zt!P+eLZQ@IByPG$Sm*zmPt&H8KgQxQxjSM1!T<3?g7i5u3b}3~j0yi*Y?TjRB^urUu_rY->HJWMdOiHhi~v(C>CR@`fK7nO|X_em+gV@3Nsh^js8}k zrEm^2Ro`MUaXhCnjm|*pqWvD_ONWl2|wuW=*YW=S&W3 zNV5O7#!P&U_fPLw718t+?%(%i5NBk$GUA%7#Tecn|E;9@nyTaaFpz1o9dY^K<-cX3 zZNv`cZ^Y})?;N{xU{ zrdo?6Wtq7NM|??Ge&;1mRp-2F)-m1IG8!f^-C%oC^C?c`*-npu!zC!bHwBXRI#`T{ z!eS!ElM*8<3D-id{Wrp8*_vWl!hp`R0(QLLa+9iQON%I6lYaU4hv5xd!0 zcBLG4uy*LCAu;SaUNw%2a~QVyV4i{+Y&?TE?S{P8_(y+TgSe%453M`pn zgjAslz8BCEAb8JC`8+7%yCGoa+#w9J7WUKmub}vGlB`G60<^Tx<7?ds2k)gC!H4@=y~emahn%ZRoR_!#k^Z;k+y-5nyJJ zy*71j8>}dB4-Ox;vw`#EFpcwvz8`vd3M8!BY$e+CR#8@mU^S^#CV4Dp-!DumV7N6V zYfJe}a-;A-iVHlatEct$f)kydj`QK1&BcJZa`)zT^QKpKA=5k8e0JmE{>Een_5|Ln z0cbW5I-3U6*2TDjf|=KmJO@iMroEz}q5J8FZ^dDSVC>L>P;VaG)N3`+-%9*l{i^=! z(vZfSo8V!%2kOZ57W1m%Dkllm3PF-G0U;+-@ypEWV!%hW$MWeX0_EvXv z+6%;)DPu?;k9-f=>ccDg&Bs1qx#XruTJCL` z`o<5kNO-K@zo!$~3_mLTL-MaN&Th4s@UJocU6F(t*?TGvLjT?y@fd+z%eB^~ct_N6 z@IsdXHtQY0g0oQ)0d*gcV!hU$U$l6=-9fajk#(yYsnR4`oKV^LZRCO@P5eJ8=UOsOmA@KGU=?>eB-6e(7YTRz zR~%~#0NhyZMv3Ax$wi(%NSg1Icj} zHOH63zSaf;r{;WR9cOY(Nl`V3yv^s2fL>3LeHIOMUnBMt1f11E(NCKF<3sudo~jgQ zMY7U~y*_>~7J!B*gzb)h?by%2ffTup{!NbDuL}oR;F68R61aufDU zW6(9ra?+j^tzl~XRDt&+F5l4b)|c75jL1AqrL^Q*hAMdU2bd$)sgM8emDi)o?L*kz zV`t&0MfxIRbBIdt65`pXdj|8<9lv|qb}Xe16<~)H52&a@@VO#tKSB45_Zh%rogU;M zhzMSw<(3^gC9UP+bxD@_0hb9IWkG#q$Gbxf)ptrR-*DB&#XoK7(BX*R^ZaZ9$@R`=n zNEtcJ&Op#iv6K*tD4W7$&@Hre6eIX`fD8dL=$!tbo&hsZ@$)*Bq52rWw2Kt>j1Hes zU2lT<0)s_PUNbxX+wuLOuS&J@@)LH`aWD8lpeo{h;TOo2J+!9Ql*3Kh z)2JIo=ID}b7$6mkaFdWQ+HB<)dtv5fCX1;(0h>s|jwJ|(p)$P6;mW6F3}GN%V*aBe z)exwT|Bx`wydHTX5aXdA?`eC;QokQRAchfd7x%Jpg$ReUWuwYQkn5)Q zZQLXws8GaOYk&T|F0^UyPaj;xk(cu%23gdSj!rFl7YwX8h88ZYeZ*bnkEb!Sr>LCy zfv6^0Rf+%W_8D?sbmrp_U*Lu@Lr|q*s|l&!z&2#3A9Up2A~b>>tZtbQ1s+#SIL(g@6l%cNhJoU=F+0G5J(WF4G?xE?v6 zDpH%eth6wv)CynfYwQR^nKccUg_wcfpFWW`f`Q5tlrf&XuwA5x-%v@fEkJSH!V~L~ zuS^IDu-u*1Ho))LJdI?;O;Vq&=ljVu0~Pjx7XQQ44%>cl5PKD^VS{0?y@?A5dxJ<~ zL+BA{mKg8F!+MowEhM~d+Jf+%KnfVt6dvB<)1!r4MgnoUAH22q5B@=|f(hZ}N+$J? zoE!Vg!q0)1TWh6ULvoi@pC84;uQd@U%*_H&|*5M;Y{f>uc9`-hO^(MAtS*K#4S4AohRzTI1C#E3G z0awPwsY!0t8Q?QIMeA0Mh??oV#?f0K^uMAwNA?F!aJ31g?@Ez1P?p2t|EdZe`iC4M(f1P9{xsGXfi)uQpMvUzr|8W zM%R6Js7|w6=|7ypt#9vT!bqgYwq`k3n)g=`0%#G7##KgMtlvK@`bwbsvS)t0^-li@ zb68b0iku7czMRA|XEVgXaDtLhIV<7@ElHj&*um-qPPSG7( zh{Ft7boH|S>oQ0z?LF3}A3KDZwxQ|F+sFQ-Z>FhLZ z%W5=KyP8tYOT1p>IHo z9=bH|7Dd93>tBG|VthFh+j`>cY9g%Qm#0b3&BOSoPR{A*(56T%Q3m`^wCrQLQ4R4vZImTSt{XozU{M!>D+++4 zdIrfAHv2{_LpQ^;D&`s|W=D}k7=3>!=0veT82J_r3hu$c31MrhrA%VHgH;z{Y$F)~ zP_xSZvp;>5{hBQ~M1k!Rr z8-9~r@6GlXF4fsqt+`3ML}?{th-z5rE$B2G>!`I;mvZzRjWjVjGqP7m3dU}5cA z_uo3YA)uCGLJ~!8iyjz6~{(^F!wn-p<*=iD!RP$0<>M2J0& zgu|UDqH$lgv%CG--SJgD8CTm!Yqq&?8=c}DqkE0#c&{?q~CvLV1v(c_FL>*^RK&{e79kx?YHrJ9T z{iz=oJDajwjGZiyZIwdQHIc$P(XiiSnnFX59hZ4s{UfQiamk4+ZE|u#Y1+L{XWuk{z-vysFy|NzQLe3Bplz~* zsXJlqcM6*=NNHdz??BHH$w{iMuP%3P@|{HCA{g*MLN0H^R4K_R2A`ENkbE-MAz7?$ z5yvz7H8~9KRk&jGUEbT; z(bDRn{-~m(adyx3b;0-ap~5O%V%S44XJgxzY&}E!5dGD+MCAW8H}TY$oA3W?@>mz} zJ9B}o`c_~l_&fC!&>8S``u?=Ebocx6GS!t+-S>3%`(?0P?en!JsMu$r8IVB!`1Nl1 zV__nx>kW$SOtoCE~{Yk!Q z*x7Y*b|@lE9B*!jW;gw|i1pO?ZK~iVH+eKS+xSIevQi)FL%vcW)H27dgs9nRju-gW zl-esH_Sg7Jn|xuNn#0ja(&4*;y5->oqwIgAi#*ZD-25+wa(?l@{NjHJiDeMeTJyD9 z#s{>0Yblj(`c!kXS@G#jv9&2}itdj2V_<|X*3merL59`VT2V}8L9T`XZD#dZ+8$M#$wt>n zPb=L>{#$rv0b6Y+yZ(9DD&DgXmIhX6TRLnaPy6?8{H}10D1t}YXX(2Xxj4p#YYUVo zmQ@$}G&nMnNVky=;ivdA9br+D~h;WpaMi_$^3J3Sqx294|*M_E) zIpa@9F;<4qg6ZQ{=75Q+dwRVVM7SLVq9RCar(*ic?9LVzv>TN zAB-hXFO0nXs1jS#SZT(um0HMw<+*7!K3f8U37knlLV=y$3Za8^mB=`^1i|)O$FLUx zIt#`0Hxc?hL4xrQ!9a$<*K2^(?3Bqu^)2e*vHMG|ousaBX@KSL4h8+Ni`vv?c1~Cn zbdg8eh#=CLo{G%2r@9AC$-a*u1i?>dCI0C;|e1_CprcZoBakx${cM5vNe0_!U%YPOafm9m7%(tS=$p;y`7`LU&t4&GN+<1r{_ zX5ta)CP_-0UEWk5p!px+NR^*Ol?H5J(&+)qPs#kV;X_31$ar!1X>W)pPN@ARPPp#u zzS=ri@9JGlnlSv3M)*IL2)V@#>el)11<<&ig^iDXVqU@?i)w9xsV%;1{+TjT zL!rNovQQ#Eo@@4b8H}NKlxhx4{nVdAn&tz`&p!!9Hcb|n-PtE+cXf=6dj|5F?*har zQN%h_Lon_Pj%Np5vvMOktH((udEzm?9%ECQ$1;_8#O{^1z5q1^a1FGG5Xsu>VNTppAzs~`L78L+L$Iss=QW1hLa<)DQVz(DDr5pIO1DAkOYOL8~$1@vBx*%tIH*n2D%ol-p~~mK*vkyTv4EBTUc(WAAv@YT#>H1EYmut zgTKg+xRghpDb7$5kkfcRumHGL{m_s4 z5zzcMj1t)mACkV#3m{_%G$({=H$OBz{@zw4{f))C{lOxxcifC@ZC;x!_Y zeL46)p^!Y?#0OLP-nU9%TziKy?7l7?6TU z+Y;q!O+sHS2uuhU^bnM)IZ>R~;Jh@qay18pb<>;&Grx^_&8EFGb@Y70@zJ!@clPho z2RaNq8Wt!=%NhSRMaMG2zVUiSI41E5%@_LT`kAn6<$XNkBXDN3IJX^de9Q0iC8fQ~ zLkNoFJE85yY+AT$anXc_>=SyJ@(f(B&Z2$A(NeGpz(FARfY#{EaHJ1$wEmt8tw#~z zOe*uY)$96wLJ$H72Y^#yIXOMhA)nUtTYY2d&x}(2E+LOw?EjA)IDce9aR2~L$G0RM zkA>wZM}(D)hc${y<(t`x%tHsga&$QA+lp3c9k02K<;X~A%x&hFjn#8)7KUYOBg|fN zY_4HB_Wd*7FMZk&;%LrvFs%BX#|6!Kwu_O+F2@X*WlB@*{Rlk3pngMib?953-n>_W z^$Dy-HAX^6x^DQiR%ZSkRAe(1Y`GnjDWR|LURlQok#0%w=3PdIB#R+v_e+e)E=e>E zXm9xe#7DrfFUsAr!7nSQ=Al}@$#kg4PFI(b#(W$~tQ=;=ZStqfw8M>JfZEQeH*f<_97%ea2b)byxKn8(p$I3^&7L)EdG=#koF1Hwg?8mUYTv69 z3rRg&8*kbfbISFiSCsvScs59{2bk39?BoY=3oH%`eiDX<9&NfZx8T6& z5`C0oxKDG{TqfM(e;}>h&NUA_$CjiGTIwxU^*A8rK~rS}%*#hrt4%3SzE9IubfIIm zk4O)G@HmmHdsKBJX3mF9Ew*b{Ur+oK4VgU|u8^D^I2h{qIbK*s)6auj(*DFW^*eawi6cP*i9!{{%Ijgh@S4tjfBq=^xPXw%(d5iAD3Ga^r;uA(6?V~ z9o65dw4$Fh1~#slo=@0aYLA|&#~!{pW4edYM=&tHJBpyO5~(cpn~_a_rXdc(cCiz$ z?-N|52!CKyTYGq4-%7!oSBNxBda|x<3;rkx7=(=|$=9@pyc-^VJh#-yKYY!li7_3M ze=1D##Oz%arTvSJnTWAJ!YBy1E2U8?s|1IZ5_1-c1FAM*HwWG^nD9H-J)^8*$(Q$c zK190qPl@!Vdvy<0?Q?mmSJ=ooM42!zDye`u*MO2*es3lP`*qy-Pu;8N)oeUhPVhOW zU>xytuPN}2u_I5@$rpO9yN)(o;njv6Ld7AU-u=v6{Q20oldr@9Kb`seDM{8JC8{|4MpMNQ6r?o6k1=oWKRj?hL&d79(f}y8(1r;;^&|ThMIF0umOm&P}4Fa+H=O*`1F07L1c5nO*bb8THMD3Fuz zPnnHa@NR;04|Y4n2;!Q*}BO5?U>CGE0K8Q1!neTL{^j-hb!!7a>q-IS*# z;=s7(@u35)38#}DNuGFj0s7DTl54%tWS$O;g-M9=2(c;}d#@x6vKbAYJw#>7a`~X#v7IgYqniyD#G9s|3D>X2?ehxPP65 z_NVKzwvr+YDI*Mv9Tpy|jY5a9zEmgKvbnayFf<%_HKLg(YU>B|+kedV(%s0KVM~7- zUfyv(}og!1?-5hm-ueuzcHyJqnaLmn^nAx2&SXy}^c$dd7DZD8`w;}C!u40c;UUQ}RIw1_Q>MYWuYH+$4ixPR zKsxTJWHZk5{Qa3;&&dHbEQ4bN#2dwC9Zx%i@T`}zgX(#+C6yt7J8NI`v?u!pKI3hJ z_@jm<^)I)qxO*wm=2*{v#{&kl6lXVYN-HB8Uvwq6F>1nG-#-owEu5Tw=a>TGC zX_e*<^i0}ur~dblwKjDic2p5*Di67ehierspq=@-#^jBV*d@>9H%rsWgSrnRkiI_YQOyQ V%P;?q{|5j7|NpJ5fKmYX1OUb6n@<1$ literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..5b302cc1df --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..fe5d5c8a0bb6fcc376d198b269053bdfe4baa185 GIT binary patch literal 308610 zcmb@tb#&ZX)9z_zcFasMGcz+YGdpHx$IOnInVFfH8DnN2Da>MoVk+WV>c)!qb=P(XhzpBG)<-1eKB7&c~J(A>XjPB2#_U!>`Ho;j^`Ub)Y| zj_Gt-lY4ZevxP`V2+5+*0F4)9XWp;p1mlQGkphp-4^Cga6W#tiI2yLuu5)7=(TOMs zD%oRy3)lxN+js;SpMH-;AD_38^pl_PzmHxpdfx6Tj%Rv)?M)o_aQc1>-D2p!L-@3@ zM_?;Jhrigx{BRh<)Zf`U4jNoI1;S@Xdj1p#iGEc7#M+IrPumBXfs)r?EMjKV+vDk& zVSi}%doewJ<@dWiyg%0WozwfonCbC!_>J=*`1O`V|5JWWXIv>07tg_)JzJ5p=4;aI zH+o5;5$K5@Qqzva$Au^)YA7*e`&m8OxIg@!jl8`@w)Q(GC03XLHzCeRU22kv-r&&B zVJPHtSkj&KJZ8c23YPNV)emku1l>b6;9-wuz1plax8OIkAQXpCJVerjd-m6 z#dohDu4282Xe9Tl=q0@y7SGT@*v7Lod%c z`c1yk3Q{8dmXwe4)(D7L#8h%%@lFABXD{D)y~7swW>4eM8?af0oLtStNEdG^Xhczs zl7AG2hM_xg<;535VNg<~7Ai|Akzy&Wr($pj8&Q&u4YdwyIyPC?>nVapE6@JAT-!ld z{57Rap}M|}Fg8spd6`JWCh8~$zE^l465Ed(hHg}d-w$q-;1@3vGl#Yd6OCiq9|69L z;t*e}Dfp?ok8b6BR)Cc}Xq$;P+mGPo-w=U&W;u3?;*j4RP@eBE&rdvj0>2M7XEZKn z6~bCDpY;)R=%K36#xoC@CPy{a${{rtU?LtcEtX2|L?uZ~Ef-@iDW3iyrlx?g2=yZ*$1K1@D2@PspeTqU2waT~1?D%eJiPbS@IkJxL&2vkgK4J-_ z6uMw2(Y_%}dD<~L;fv*%1<@mad{n%V&#NfwSeeTCM@g+`lZ|SzM!a*> zH0%S)F{xKL`05)-X&O2SYn`F=?7`7FVut3o-+c~?*^g@`km_WT7a(W%a;QFRlW|06 zHsh>bR{cSfw*gF7*;e^gFOkE{Vt!ib4P~MWyTkk6Qnl$x)J&vJ`}31hj>XK(YYVcfO?-PaBnbuKqiL9E_dYC4~rGG!y^%iK3T zMm3|bTO5W>|N36IcTf>H0TX~J&bj|MJ?m7W3f@j6uD+9l?h}(q6(+%GDz$bhdX|&e zMmB&;ULhWo^%XQ*^E&;S-Jdl zqt0HIfrO}BZmQ!DI5F88+5-p+D1qq?no+S7j6OTWk790L4Ds3RttG+78Zu-$d zXKWC?J-XFXkEU16t{H1{T{1#t{NM{dw?#Ghd~&f5BJ$!IFWGKlkr1pzI;yPt@dh?x z%`K*8O)!ZKY=>9aXXB_}=&KpFx^sDq zFmy>Z=yt-e^X>XF$P%okb;&mTp`b|%#nOeaB5}1~xhmlLD+xr*;|gMz#VU9@Amy0t zh0byL+2d|QLXy0xg@W9XsW_#KS^8}x6ci^c-dF0!2%mt++ z@1S@28$_5MKLcnd>6PMB2|k{@AAS#Q=kUJ29-J;Dy!#!mn$P%tOtRkyL=rtXQz~LR zwI;e>DF8h=G+K35*hlSFuKgw(2g9ESJwwDe8NzU?u1xgqC9@?|>Ze3$iE~`hibSQW zJa26Onl~^UQq=4KmHlz~>t*}XmFM++JLmoP!_an&c+ThhYC}Q~-jD9ap7)F6l27;B zp|RUKx5&Iyn~==Tbt}5+jSO41Nf7p95+E{9XQyUkED!O~s`*P|ggeBiq$kbhp3M;l z+x)APq=-TV$uhCc*u+W!b=A1NBDIBU>aXzSf#w4N)AX}j48g|-IC8A*9CcsI_l`5s z_$MO5a;_z`kZv0?)pdUJ96p%)n-1j^xm$v(>@4&Ir7B9vPG0@4@a*nlU8;GiqyN_V zo}r{n(CJ9;EA#nw|LO7iI2QhL3M&6Ze@L`PO5`lqp4_&;tis<@rG7k5T)Gr!7Ta}N z&gdF>+v7G2uy(fXY73d~d})SHg~jyidXT4P|N4(b}sF{Y!^<=-~c4 zlfnhqdt#-MTnC{{M?cy7)tyU}T=@e;yge}@SUZ_CYOa(95hHDsmZ5pC6Tk7R(~|Kpu&E!VI9xN_%ALL6M-0!-GkH_5T#$VGD$_CxW>ZbMI5h$*Cb4Qj!}{! z^`Td6?8fG6#%yT((>oPC5gBj$n7eicfsRKTzpZ~NDCA9!f2uN6&&51M}E=LniT-LO_B!q z!MH62@tm_y@X3Y8js`dK6Bb#U5KXet#_`$2cp9{QF{Rr)IqS!aZ75ZicTbMJfIYMH z)6;W#25rH01_nbA=VK5}r-157Jp)|DGDOd?oC*}K=cQ4o9u!P$c^K$JLcHnQIV@|K zPJ$}7Wr~r*(Q%_O$8@;PvgRT~c4VGM`jr(7wlR0w0T*!|2zAG4j`tQZ>^4rhjO%=W zczNPiGeHT{G;<|r-FlU?PItzmP1lnVUuO}Ql05qL2Yfx70Cuk%Lt&?^a_QAyZyr~g zqud15?96^SmFMC_ZWM`JEvw-zr5+7{9mc&nIHAeQ^5T5by;=1fygChJv9 z6K9ba);QxA)CS5=5HgKeWLhvvuq0=};gb=N_g^BgQv^9+MultcqL|_3G*~HD$*E$l zV#TfF2jTrv4CKE@$PL{Kro{${|`Y^EyawT`9x8_9=}e9 z^J)-fT8Qp?#_v`?{y>3s7;)dxw1$nvdA^e+$uTdA0K4LxXsDdur9i@ zI|D04&6R@#E=2`S^cHZ*M+pke>1EZs(Axr~cP;hD0Hue}!v*P$Bw(kHVH9elbe^)6 zG)%UL?2RUOHRD4z`awlyohqZh%iutBEOuqEO~e=z#(>*TM9#zX1tW}+3pX?$T)9?Z z&Da9Lh9Fz`cG6ng#Y7zB1Ou7R3+gOyrx!m$zktIK72^PQWw{xJ$pPf6PHPZ!GDV&v zfj^U}=I|Arq*U8lVjRj|#?w0ddv~2RQQlV)@C#ag;RI-A-;WR8Jp@0CRDSEFR1^++ zV)5TsDqSk#{5Z%Iv`eJI%ESq|9Jxv1({tvwE|5`-Oc5oNyxY@}M|xYqhr=99^AGTJ zD^sn@mH4$*mpV6$w-V2JMg_Zwtigj%d#oD6H^^Gj$rX z1OxNo=i&1qodYNePg10wEc^AMlpBluBbxOW+=bDCPmmt_qHy1-8Z5qLpxKzVjotEmGctp%Xme>xo(<7ZcJxDY!#oPPY@V&#z3X= zGQ6Q(gs0fDe+wipMfJtY zCbWNhRsy&E6dyF`yCgv9z2H|eL$v+0EywR%&k0Hz)>Ut}VNx5oh=p4{@_0w<;I3sN zlZ_%{ePwU~>D_^XVBk;#VzNNRPlf{tEJ&eZ;J`-?1P>&^{l4l~IEfDz12ELyCkQJT zACXX@t>s%&fTEpBV6Jk4V%uzLvj8`zL)Y&craPfszriJ0&fPGkXAP}?Mmg2Yf=37s zGoMQcZv{mNPX+jsQXx!1=ImNJ+<#|v5m@AJd%hgWjJA79x*I8dN!gSSZs>({A_mJI zdQek|(u(h#O#J1%ISQ8y>i?DI@n)rW7e=WtMWohxcRl%%5AErKu{v^2(hR1Y9yz;K z>P)T~rr@#wN2u%*SCXn>b1HZE$v<5+-RQCl@gf(MX^e5InX>Css;f1$|E)$13_fLZ z1#zq8n4$XMw~oF*GX9W-aj3#u`lFEu#tsyZIn1G`0$$J0agu77oY0a-#F`$@Fho;z zo4DO^BK)Wz0h&l3506)~hocMWnas5g`PnZ+puO_` zIUxK#q`x0Jasx_m#M`UI+x45Y96~Mh^b3PXGK(|A2p1NmgDtC37TT{fBF?8Qw^^Pt zzRf#ICyN)Gd52r7pSQ4eY9mcsN`dtc)embp*5_8DtfD{@XHji7_V8F(>Y0&}2g7Mp z{R#=#jz4_%46A5vlD8YXn_aB;3o%iBoxJB5@yMO<&XO2u4m=UOT}jCARPwaV7dJfO zbj3vu3T|f=#!V+-mdH75Tc56ViZ{h?`GmkJ(@l=oB_Nb@S( zf$;Ea^S#&dg5K=ML0I7VqZ5dr&>F zioHUg4-gect)@YPFc6hH5E-H(`1VorI-hO(P^Ic1d`s!N&Fhv{@7+ZkH)`YMIi$t< zF|fsY3p9b`dUtAuMP18mzlZKkGfXB1Rz|eZ-ZERs6hkkuZifxYDuU4}cWXr=tfW0% zb6eHJ(V13PyRSZUmUK&?D7AXK8>ZiHP%iwQUsm3oksWf$<+1B>b{73<1`ByZE2FO? z2PM2JPgKZAS;@JPeI3<5rc8gT!nDaU{sIIY9;Nab*X_>Z(M)r|zZ(=Kuytq31XObW z6LKU}qBsl2#)qEBYs`xHa01cODFsKMt)lFL7pJL_1P|&^EyFYIh~+L>cWxU>QPpd*~pwd{b*>JDP#kn7^BIPapK$aj9zDsIgHyA$0bW+_J*o4n|b*SrnuIH4=xu}bgo_nf`^qmX% zY>@Cf7qY+@qzZX?E_E)#vU{DE2V!Syw=qm7raR-!9uSb;?^X|oDCB(Eyl<}35mkG` zz^00maX`gtkB;S(!rsDNX=?>n4eGoU~L z$AJw0XVo9fRMoVyWHKC{+3peF*xk0_LQ@7g7Ci{FIkv!fw-(8z%8<={!www;*4^HJ zH&bmx7}D0X1>BE@ZIqk-g~h`@TMjn|6z}@6xfk(+N_BUO-~Z5qPIIe zmDSa3M``X$W_!OAo{sa(52$*jhbbsxsGzM0K9q+hJyqqXJ@}2ZK#SpSq^8j8*Ibh| z{oY+pd4gyEsywk+n-)bwg~yq}cnf>{1;Lj^0hVBix^d!0Vr=#85Fql#@4wZ|(w-lQ z5DTd+hCwU(Y-5`gkzIM^-B;!Uzf0BOh+Nfc(X2Ha(SWq@x4wslL&z&Y{pR8k+6$SheL&Dq5$*7V}gQ< z(JSxWaTkF@hl{ZXGW(2U@?zll^K>Cb*i^sIWm5qc&@MCO6W>6rc9-|&5y__8SaJzq ze<>6c#9l-az*YewBWMg1cP}z*6m8n=WewY_e=D;`|Fy%*P2v}uJQ~LvxQ#?~6P23W z7k^+HQdIJwX%T8b#XK{7Rw&ZbqOElGCoijF*DSAqsP|gUVMeV60)zhGRb%_9G%KFj z#n6XOKFi;kx+Uxrz)*+FxUg%)QH?K{`!fQqV|R~DX7Y9DTP@O0xy;uwT-IOm*TUST zLj2B0THm($ETHpX`$w9f70@2^(F8emWuSr@eA8XnD$8!F6iihfLTEwedAq!_lZ?b* zq?#!kKC8d@sa;pEtX}saD^qXhAy6h?XXEWPsRu1)R##GAE@sM|!!)MiQJzv9D1H+V z&uN!@9sAw=%x%~YL!SNwRrV5p$SD?(m})Gk)J5s+5!MA{)9Xazz!cs}{bOiev4k!27)xw7(N4R-~Un_}ZWKi*zG<2Veq>N9N#9Ed? zETLH99dr`zeUa7+iE?uHV50V}mSb~n*^f4P>gtxRya6uKesL^kdn z{U@iin)vDU&oi-=yu%~-C%XFN^Hr`$yc8c_l@mdwXJj8JCV56;NCIjV5G}&4 zolGE3J*egrjyr_KVI)nXUe&jA=JRK${8~#V{HWy2z#_~E-JH@V6~`TAU<+o5sgTIi zmXqBE?{eYQ=ml_SECLUKh?|k>g2OJ61P0l@qgVuj+tjiFtr!rRnZ0l9!ILMxC9 zWLYSJ*wp62qu&_q4E06|Z~aQTF|r730$53d->IBXKow zy(gm_^7<*1O@l}!pbr=Vax@p-UE_ahXpnzdkA(oYRQT%y7!&Yx4g%Xw7r!WXXkoM` z9dwCcO_fo)L}vT>M#Xa;al{2dHuc;gv>Gr)RC8raE$^vbUkAm;D`ycR3r+_6fSZ~K zGWlRAKX(XVOeTMwExPrNLIcz?vI+7dF+xxF?uDIihw|tg>~9a{Acn&!G~-b4l{+?? z3^lu~{aE&uAd8l0Ojm|R)_Jw)@X(^Q4p>b;76y^%)J*5+I)o;+aniSN^Kq1s!yOvS ze;f>Pj8qQ?AFg%K1j+?sI)7`Po>7Ad&Pu|=KtfdafWXGMjBK1G&ahE+){I|9KqQL% zqKbc4gd7ic+=m!asz;)0LVY4gjFNfkg!RxWaYR~Gk5tz(GylTaG__3DPRZAN9F49g zqEGb5)42co>)G1b^5;wFsT-xnPy%M!Cwbv zCq5ObaIQ{P9hC}(F#Voks0?KoRPZOUl& zlnfD_1a4oZ^B>?E&CjUzB;5r?mO9?{|_5#YLY z#L!+3x^-rms@YLZO)<^@rfd1d_nnE>%qk>Wt{W!{)l>5k)4sssjNy zNEejCUr%jz!U{AuMRj{&cA^~OPb2)(8pF}aW+z}gHGu&P0${BJ1 z=n>F+xh;*>`c44I2wVI(fEqp}(s*0nJIXw7Z2eA5(S*qUh~9U&weO*1fxSB?MKEwN zf6~w$C`f}R0?G!+zyd%wU~B)E_5U=(SO~Q5fIAT7(`ZMzJ?6|tmdEo3{l;pq8wyd5 z)Y`wqg(mIx%_1r132p|x#5N_zmJJ8no4@<@L3CtCHv!rZf>a2u93nj1+?bSVIv_Em zNzAyNuYIjOCXJFO2t#8Ye2#WBfURf&~hqf=`Y%k9frR~^%iBaRwN2QPw< z0&RRAIo}5C*NDxZSKo)K^w+Ig4Bmn@rS(X zKWSZF*#DqnPN*it*ryJwB;hYdKRvfM)WGN|T4f~b6DqQf(~g*@1Z^qQO*WCC22Z^5 zt(FlTaXJ=8jQ25|C-toy!$v)W;`bnA+PPPycw>Y^00KciTa2ci!c_J7)afxN%8iQXAmY^pUI?}?hN+ulavkTopyissdH*e0Lx z&MYI!#s;wu;e(ce$s(=hL;Nc+MN?WBc8($6$wu@LGD@IOGPyHIzwhayDMIQ zQR|oXvn5#w>R4G@uXk8EF^SNk=svQXEv#!woHM#JB~RyyWq1*3Cx_st`|E?L=SY%K$dK=W}udA!hjv%M1gCe6Wvx|!0Btt6I~6^KhW_{ zRv{9Ja-s{>1^W+Zyc@0w!VtdhM-#a2$If*#>Ze*e9Mu33yA>9aYLHK`^mhfOPAIuY@8psnv z=vwG{;s0{=0^~&Z1RS8W>jXT488@~6V~ETDKnItH0*AoBApl|1I}k5uU=NUbh!GN_ zS_Au_e{{KTPIgI{o#U?u5yV0v5g7321Ev>n2P6cIk2WM$6Wx~-KROb$_Q*YHDJ zeGZYr%_P66gLDwyL8Uw>hk9VBI1+onh5H%0g~^gJq9L9U@0u0BC|_&F_Pe#KAEKl$ z*ez^O3I1{?yCEHidRhkk^;;ucM6EE}<`t8Qm%q50wvx)o!aAATT(k<`f+b(kYEJvC zUrmbrtYA7x1t)GHanb8UfI>5{LI(Rce$-gzoJlGfM0HLf(|S7OX^LCFzq9MQZN{lR zQ-Ki5720+y4KsH>=t7mOGV|!FR85daRJ5HAEMdZ&N5R|Y2|-UDR-2&8;u*}ts6RgE zEJ+FYED7XNCvkkjdXQbZi)#I6@QXW#w8-)N`+!?vdL{1rEn>T5pZfm3elFu|nVJbx ze1ecsbwXQQYC{LeZWJ2xZepTtIu6=QHOW$pa=|wXWBb4z;eo`fQue(Ec^9$GDKM%@ z2{$HE!C%wsz>e{=@vmN_+TcnpS&Dnq0?$xLTG`pPo9&^H*8=iCUvpQ>Sj4A3gbCOX zZD-|hzq^90jd6W<4Z!XsjhE-1UaQHs9xLtQLG;iLb}J|FQekkNJ(oA(Xxl8F+fMRy zK-B%H1^}{_Ukiv=H9a}mshcd%1L4J7QbXff3)YOOmoaT9X2FY_Og!B2J0|#ZGju1| z7e|g+R(I(W!Ew1mG$gdI?@@B_@?`kEv zIiF*rg!E`qkubk{3*(ilBqhXaHlR(iAw^e>g{D21J@8RZPdP1;OYNL93!%5`=`pvT)+hO#^22-ua(RpwK^uvktSpYRE@Ez!9; z<3&^TKUM?q7v)>yjd04YBsQ(;~{URyPS3N}llf-~xB@ z+8Ls5*RT6#7q5r?Z>8R)_7_I@2Yb~$LHdUeI9Q}s^ODD7d`u^Vg8~zDWkJ%%{L`5(SNXO>~bguL-+52={Q6f958AvJ>*;)MdjcW_8 zA)o!VHGC7p*!~DKfOb5a1k1+tY-s_81O7%r+#jOAeB>jAcsihktUOG*3SXcuPFg6V z%}w_F`kH@*xbWa}EhRfL#Wx-Is#*cbl8%~*(yggLXCe>ZSgdEqquUaN&|8s8j~4b1 zk?PhWBVA?$-$%8|5_wBeJcduCH*VgSGcOEEIWyUKx!WL$peQz2!{MqQ8 z(G}jm{AG2j>9ZWeJ`R@7u}_q)b#y8J095Y)#)Z13Y9paRjQ5bOWRXIcGsMVfSS|Pe zT*BgtuZgO79S`~VF6L@xtD8wt*{*;7L%#p!>i7y%2(<)B#EQO&xK8iURTvx^b1OZ5 zem-jW#5>}v%yg5e4Wb$pxt1cuEiH2C0jZ)92lP)I2cyP8p$*1vU48!NJAAHs?#0Uqi;&yAwb8jS`)t_|DA0rg#W^ZIF(l`pq21d6*TPbK+Awrgq zF$bzn{P9pA!7~5N>=okR-QC_8yjjORB{8OcIjZK|IcF8|Aws)rxIbc zS&PA@BS(>2Uh|uY6_Nl$p15;^2Z1+(Uw)1U#3}NuPgqk=Om%g_PDI~cDP8Gqnt?Lr zeC?rRi+yl^%PszCCDD*oJr;8pEQ=k$`)^lIv zsq1|9RqQvr1+Rt9EVOLT>h~W-{Q{)BT&+P9@n6cQacqCPZR>U;9K6L%d__@%P(QW! z0iagKEv**g|1Z?axnx?oau2P;^Vqs3eS47^zUlXJS#P_y;2u19^47C(mX>we%n z&}%`VhtPmSdexmLKLC~5`1mJP2OzHEv%ZdZNsYqCdFQeJ zf7OaYs#+$IrVfcOLVq@Ol1)@46t**GTnr`O6fI9BHpg54lcuwR0n!5id6fxd@;_-= z*X8+`-a=%Vv+MG9`f8%vMasiS88LBH(yzG_$YMBDk@=o*Rj5(be>8OfHI%fEwTf0B zh2CPN?hdATGB);N8I+e!JS#_OXmtk(Pd2FPNp1=Y3a-w|yizCAb!z{w=AkHzsA>SwhLl z%hSR3)&Iv>D}Fe|U@HnqKrn6>=05Ic2HG#VDAeS)!s8c}dki-hQeGAq^?m7frIxqD zrPY>`7_;t0QSEm#@=Lzj#7sG=Q&jGCQX|NT=*PO42CVVF9C3xfi-C~ z9YP<#`$QKT@^?Tu06q_>7`}ol0WJvfC)tGl!+)$4L4wZ!?f8#-&z68wu>aWmQvM9m z`rdbmn4JqqV*kftCIQS+r~)(uBqlClAlr`N7$F0PXBkM~J|N#^;OGZVeHS0U=-W8~ zbOt!Me_0p)8>a%8D!2jQ4&o5k%A4X`LUpUO$Uqx3E>$a_?&ZqglbzIJTU7ao zcml>@AJM5EzJGt?bPBfc*Ne{adkkI! zse-)Yc~Nb7h{2a3sV&LKIMOYYVcgZjPKk!XT(lbD9>l49=`dq~pxT$q=LImLbV+`@ z`}JJ@*qogRy9289Av)JP1S8iQB5?)^h$4p(ip+c*DCVFZ|cnMvOwWR5UB@#B( zY%E~`px{!v!mcQchOsgq2kfCCb< zV{)7_KCkora-g0viPBzE0QK`$yP82w*QFq1tLNk0-I9wiKhPmiBB%laHqdw<1P{d_ z;SF>Cyu8TIeDx#v#JPY}ShV_G7Rz=*o`M9pINXKG?kx-xowtre&Jv-*b;v{Od@$2A zLrznknlYi06gwO}^qJCS9|i=Q3P1JrsXf$;3^~l2j3MqvGFjjLP6fy-qrqEFV+svO=&ZcGK zFWpw&vUr<2DvsM5j`8SDej0I80 zIH-r%viQ1J=zFM#@U?svTR}a2;ABjr&AeyK2p&Z)E7wC?N_4HkQP%I52iBr$hS?1J zSF6UC+6Ewt)DEVSl8XB+p_R~kW57Ben7 z9a6O?XLBC0*E;i1;t#h^-mwH{EqwxejcT=ZzM05+aLAY_Ab6e+{280j?p`;MGxB#m zBx2z2ddL;qa2N8)PYv?5@!` z-RbHR1{XSdh>51{7HvQ2%pfqN$CuvW-g^#MH2o;F`XR@(cgl~-v8vsKqR3M+CwCwq zAdGB5p#;kEMHBc}ci%I7x803T3x$6O@vMRgXl|YafWs;j{)@vZfMG`f{ogrk2r>2d z5MovV;W%Hw79X%?0Ktp417k=0WqcmoevW|dbzpM_AYpF&yZL}9%YN=p`J(mYKLc_w z{!r5sjQ6}eiGOj}KW?#n0eA!8uz3>n|Ja*V82qm}tOW(if6Zb4JPMG1${~MoSnLQ9 zK8tlb9k0l_Ch+o)k)T!-P^Zq^sFsM`{1STlnVK2>?CvML2Ld_z?d+cSdm=8sx2X%W|QYLQ&OynD2{k3&DwTy_G*$^lY8krnO0Z3wjD>U z)|C+^P$ZH!wxb>9PN>Q*zl9$-1kJ#=MOtRcVEN1|Q;F`nC+!XAJ=cnttc%Z3w3%6R3ebx>yLpM$Ac5#S=Y;=_eL$35Tp<@1-{1Cah7GAtNk2ZRshl5AG!GF`O-_q@L!C;t4PSLTV`rzHAFet( zX6Mt)WzjF~!2gbMRxP~w$O?o3+zQnXqyx4?{Js5L!42=Mx(0Mn_wR_SALviqm3zhY zcce2l^qyLnZTM<_Q1g-}W+GS-O=x@Fc2)Ssb0487n*IU$m=7V=x z@dEzKNjiXMdO&}Q+`07XTlf13~{-clihY1+bI?!6~@$-u3acqcm0N zMw5s)FtC16A-NsiK+qo?4dTxyxM^7) z4QpbN9wddd?q8fX2rxJ_S583lpZxk61Tfe`Dwn1y)S5yX-n1+fcwET~unKFQ$$qb2 zWEIdna{+Aab>XTb;Zfn)p@eo;90Gx=BOjH(SBCzSM9+=>ksf>9N`UnE>cWjHwfswZ zG|$ig*Q1cW^uZoidIm`>3aMYz1u`xB@1>`H@xPXyzpVdDdT3k@0y%wa0J$%RcLxKj zulzlgxM-3*)CvYXD~wm0{@a68zpke4Osci!3FE7w9E`Jfn0+#5kcLw?X@yceg8Wh9 zE$#%W2<*#CWFNVc7R8zTtV8ebIidJnn40HG`l(jB^k?9484{HjrV2K&ddaMLW0o;j zjO*%Z-GDT$YsU66ce938Uf3?ZXavfag6$rx0=M8~Z+L4cuZoCsx6P~rv;w#^+G~dj z=PJ`6`s~* z2V$dH3`J~mY40~wXZTi5L1l%=%y`uS!%KTmDyv4S)Ap*&GD8m)$$c=huH^@ET$H3v zG%c2eH=rJt;KS2z6rGil*tP`**|C%&$z5-dya)UkPtyoIXIe zM>tVtwZ!3Ei8W}cho|ktJzBY~N-Hc_> z6Km>FaxP6lx}?GHhE31SlLvJF1QsXzKt}RNUI9NBkOVF$0a4NkD7+6F5ccZ>FFd@@ zze@M;KDl{_|B8}M8bB;gltEb@P9Xt?fGtiMpl4@GZ~$LpdGHV6(*+7<%bkVT=-LVj zb$P~yu>tV>V75Q+sSwV8LL`enH9I{EQ0&*nf4!%|IcR^o=YtLK1>_%l@IL>2={|(- zzkW~uvi`64G*A8@usKUk#AbCN(kaErq>_ea1Hpw1RA?ggx=^4Rudk^kQ;)D)d@)bF z9(J-wd=COayZ+n7m<#mqLl&VoWa-tWj$v3;bYtPwuYn!)x1JGAGH$k=keG!^T|+&l zlF;Qgq2@Hz7*nR}Z2STOD7D)l+%Q#ecwMGh&m~0UBtR0Lw(LKjxF{LNtb`O}S|VVn zxFToV4`Rqtq%*20hEB>nXlHw1ScFU-PdCasHReGCt^bfrXM3MH)qoFUcx zvC*LWAd~imoU^2V`m@|4F0cB{9^tE$=kxuIm?{+?Z8<)~ay7qYD@eX*IVizU^WxLyS$H|%O=l30RUy-8Eu`iF zFjr-vxFS9w>cFba8V^sjFLb-n^J>E%HuXKQr26jmjv zEd5rkB#lqA44$VeoK`qf73(OhQyo`96O;YFDb1aq>pY+FPESN-=923l$+yF!%_N>v zz5_4d+X-$smrKq$f}8*F;p68=UzaV#N7yA|I_h}OHSFM$@4!+d($%saF;3@o%Uyuu@hTfIOT^+%PMOvH>EM#Yh5$n7- zgOVks218*7MYs<-arxt+5IjPVpT~&8R|&|k12_7gD}nERp|y~fzxj?ewbRevFPmp) z7s0kBp@FY6S8?N@sJgQvF4UAkB>~rDEt?ArN zbvS<}>`{cgRaZM3`rv#=7edKI*!m9LpyopPYqW{=@l;ri!{F4LuUanC#N~1i+}@5aEs&6105cy^{)z&)W`+nOC-PM6jvbfS-*(#&nYXu$57K zvWc(CLLN$ou&kS6Ez@PViy4WkIrSYvvnuS$rAr}z=|Xc?^mpWJgbO{;Q`8tFAGm*W&X6;O~Uz;1R6fG5G*mK8g>*xKS6IT$0H z#+t5kTNwLY_4hNSW-7)S8t?O&MeantxI*~rZ6iIsIm$IG@^uH>Y{HnF7>{F^sGzlg zcTkIc#dgi-C7bK$Qb%|BFjT&Xc5aH6_uQT@EQa3;GQph*d!Moc!p9|bnk?dwaxed6 z?uTcu#-w-1-u%QF0W5%-^V0X9HItt|^u6;wKYv%$_`3ZDOsMzx7`pEMefDxs-~HHayxfl7j70ZDK_=<=oe;6)GhdxkDLTg7QzlEo5JU3TyIHJ@!;*|PYdZJ zLLS3l(%{92h4PoXQBDFbzZMSs&#R$dzrrx})*mM+c25XGZLZ@Xkcs>u3(NCd0HKE7uq;WPmv`?Ek<1{E`BjR@R z4aCmt;56V98)!(pSxAO7Znq>w8xrlKH+(!Dczi!zerfb5ac8aTmYthVv3MIrD=p${ z9PjWedaKSAY`T+|z7gRB*E$UQ(zHgX7JAnMZP>q=lrz75c8YwCMs%KTb<_+s|p}iDWj#=R67)X4~{lS z_0cUe#)D`5q||WmPg>$rW9Dv_U++mbed&JefZyG|oys3452}za7NHm`Z7hiT>hk#V zI8*(;>it~*E`4vvY9{`%lLD9q=6KS9G9eZ9jeXIS9hyTI!P^g2zaOSSV9!pU|NV1# zZSQ4(V78A;QX_6Qst|JK!h-0yodep#2AeR7 z@{3hlwbnZF)WT`?+)D;i*Cv@&)}$0!v^p_iyaMX1JrZoeY4BYo`nk&7&uWJiy?8k3 z`Z+woh7u5Mf`vTSl_QNT$xEP`2nfFSuoLS-zJhtl+W*Y3?+^Q*X4n(N0%q8sxn^*j z%$~8sVbdZ)s0l!W8=q#VS20&*TbRS?b+@yDmq^ zp~L#gfD7E&q>oRH(k#K0gs$?3eY>q)u(90cF})#m5yE<451ZN zCeieBAK-Cn7u@ztI}ltMTX-eU3`&pX)KeJVyoPL-?=aqA*2J@IT z$(EQeK96#rv)@9_{9cXV2A=OUcZGoAB>tr3;a26m8!sHbYK$Xid5KvWRhvfBlaK{N zhB|91u2x7Q#&Rynoefu!T0=_C3wo9Cs1lw2Y{(Nz*2p~xd3UUYM)SwWB*6!DQ$bq7 zL#5KakE-7 z){8%}iO?v_g{qIdfymI3j#r_0mQFfz7CLMC>q(YL@~(6Y(RP;xjf#eKrkDd!qYy&r_J-nA^4r^6}I#z^O` zJGG)8_`f;J4>fplLMl*wJzK_dLItwC&ivTPZV@olX8AH~+0(zV0UdwCV*nAmDse;{ zgheow1TI#zS#W~MyOfbQ4w{F!8kW@MPYzmAT58qJD1b?&@ow`TCs-*pGyw*iPxQbi zZyD$k&K}v#`)n4P2Fo4MgPKi1OamreI(-K7lM-TWeMY8@K8^3Fe*QHCp^c`NO$CGh z8)Dm-6^K3Y5a<^OyDe!{VkuXIYWYBouIK=U{NdN;ba-s))7dXKY!Iq9HX~d(QUwQt zVU(6NL6j=X5U~Bvc{NpOA{FiL`EA{1M--TpR)bH^p8Qj?kIDO8l~V!@vylBXDO{pX z*ve9Z^-1YhG#A${|BI}50FEu_`hH`d*tTukwr$(ClM~yvZQHhO^CY=>p7(oi-LGy{ z&z`kFKIo{qNtl-;5_ustWIO*TpVqm9F^d_LK`zB%VP>uuEV1!7k*J>ZT)- z?};tbY#267bFgbiE)8M8S^{CU@n#S`%|CvOPkwy{ED~Jbitdb9(#48lF{QF<&|Z0< z^paVpoURqgD;lUNFQ{&>kUb2P0*+&@^ zc4jM^=;2c=#8oFLo1{HVn6*rJ`7Z;ZX368ioLROFhKyLGT-7ZI6lj6LFqR~J9^^Q9 z7J||TL(2+zSUF#GQxz?|Qnb6*D!#A{3Em@$(KJaD;bX*@h#LTBYzGnScMAr8E`T5W z0^~neARLGQ*dsq~<0OReRytV&Zgqv3CcNlehixxay^Za=1y3^(3SKo;Qa!8HtZCJg zwlsvN-*fA4Ypq(P(WmMc(uuQrAk)@NWW`v5u4`rciP+v#Q%`YjT*lpBo27J&_O;}R z$K5XW`6zAXp+Ox7Pcn51fU?f0Ub4gk4ac@69OXtKq1hQ@p8&FG$C3zC*46^oWe)hu zz-ia+)XR|lf_g(~aT$&&RFW11I9jU4NNkZ_s5a>?a(Sa4QkS_z!Y8|PdR%)~hWX%)?6S0cRp+jS4P?%VuWo8gS^5XRIDlt9b z#D{U1mQ1N%Z)k@-$$_7iLMt3B@eORkyC?MNY^j4^wFhARqW72K1JGX#lIay?TYLJC zu?yJinu&XWnFqWKz&Qut4~xwH`#Vtk!-8WnJ=!N-%ojk zqVVCvZ6p+H=>$?}aN2{9u$LmSPN)ZZQC*s^wh0;+vH+cqJo<)zPeVDD-J+7S=64^J zocWF~lpbiv(XwylpXV602?4vA58TX6-N%g-xz!<@-H48!Bxit$DZt$F@#F_Rce3K$ ziNO!}=QMjU}P*1UZclnhUB-yb?+~b zvQU~YN7po?|NHPktL3g$bJ~5?X+pUKxnhMyumSz4B=okhOj}ve@v-2k%7$(ZSw*6T zsa%b0?-5o})^Iid#%zqoZYwM~ZPJGli<2T)$*+?M|5H7C#+l$M!OOGRA8A+WS3v}Z zX=Im)Zn1Y5|KW4Ga)BwF1sXBvt;Ud2^WymarG`Lm#I19NW=T~pYJw8e@<;_mV%a{? z;^?3M$&3)~5}f~RVUB#J`x+@CE4&!Ca*Giqyzx+I<2g$BS{#_dbuC1#SQP|QO&9+> zD6cbVI<14s@s{jlGiTB=jIid6+Pq~NkIr>=>5zRLseH-Xi7U%~8R)yaI8lfk;Yjs^ zv@rr6(z3gH-$VWcxZxxEQ9S30BWNU6gF>+AN#eXWP3UyL;1I>Ky|iUri(c(=)TXR` z+MDc{4>$J{EafKMF0UGVkHzM|ZYFo=)NhU=qKx01i^{&}Qsr_}XoY#Gsoe#|)VXQT zW;S5XJ7QS^#M4ePNfvhLO@V%x_;?TjNnv}Jrx~^=W0!N*%LB>xrbB6;W!ahR#`<>u z$-<>=6Js~d$UE^>^4zhQ_k1HSYSZ?|u~N78vq^I_>^@-4K6I$Hh`%XLT8*%v+@=ws z1*(OiN~3$C#e53Avs$wCS5RowB(9F@*>Gt9)$DQIf}750(J~+PQIx`AFp<>4W2eg& zbBz@Y+A+`R5|CqgG;2$^xq-=sSIJ!Hz+;$UTXJF@W^knu-(i)SDXt_%>CTyZAh=-G zEM^@2w6hSZG!Y4GQ6|Pro!2ZIu9eIwZQ3bMW;-Hd%lG=iG|C!Ta&qW&1i4f zRY2Ogv=66M!TBSv7iGz=eCeg>DRj0QcF%{kT@@)hypE!fk_x)brrmFew;|rFZWXim z*2iy_^3be=bAwzG6U7^ zB?8-52AgIKVH#sLl^aG3;Q7{?gVkGy8@je@+XgN79l3^Gz=v(D^=x%;o6~5}T9Ey-19fe_DHn;|oe>(mA(QcLOX4Rz&5+IVFcFnr`N_%Pk zH~6NL(Def!d&_oPBH&&}g4-vWTO0AQ^He?)m>?Cs6)ej>5d-MnL$YuxGolDisMORg zWyWr;kCLud>z{wC;&kjCxs)530G6+)n=03Or@|oUAOGwo~H1<2?MZhcNT-&m<~TKsdaU?S&Z7-C38Cab4`J@Z4Hnob;?R zJ2ml|U=C^o4H5S1?DhIyNCH0wwd9G+C({h<{3**&&!BElvOeyL1Hph4;o3~&b|f}K zQh1#K^Ke2EUGXHLAc)p$Z4d~blYk;|-TmImU?D{Iupw{+OtRw0@I+0r&iH3v1aiin zH>B*%-~zI{L0tlYIZ5S_uEt?uPC!OMThw+4U7x|r_mET5aeacsO9R2*4qg*tds3C3 z%9x#P_T3dkx6z>zg8ExSFdHvLFcaK%(v@z8w^E&W)8uZ3Hxli5)1+?L!aE(O2kpMy z?=OiTvv)rtCb4T92fE$el38%eAVk6!?Opb8H4f(4yj~nD&2rg1ak-$k+lGO)pAhWl zYc|<+vvgWryxAPO#MwNmb5lLzT^}|GbT5YA5bie4HrYShBamF6M@22V#D(nks*GI# zvJG18HrT8JTW<6n=S^MX&`$TNGe5V=0^$OIAu!wl1i^g$;{bzU@Qncof?)mQ`Z4*( z^#9-+-&I0l1n)kFcb(2N4i8L);;4z-DGQ%@RpM!`W* z1<1C?CF%_QtwI|8z4o=P0&?#2lVShf?I(v2#rW^E5-JAqA4h~5w$DR?bsh+iWe1-14$ijMbo24t+=uzoKtN8ku@9r~T#v5Dm8H7?Zg2PW z7I7BAjFnaXj=ggu%bjgNw9`WCmchO#|+Ez%Pe=7I6r8 z76A-;;|UH5dXv|OgWPrrN^Xw^Bfm!rlh>Dh>OgMa3Mjt^3$^tO{50gz_6H{~bFlF8 z1c1jO@AngaLxAu&_(uu;CloM`gX(|lV}Hj+aDMRr4vb$91~?do`>)WLKJb4>im!hT z`y5g!?42Z)t26ZGkob(ld|U!cB9nG4`0+{E$k|jV zS(&+^h^Rkuz#$jOYvG|h5?t94LPWH1fH zZ8TlBoPt23P~rVAHPt0^@b?w6;8 zliMx+1v?LqRyKN$adWa|uj~2LMKF&|oFbH!s?|>A!$K&8os?-iYLJ9LJDxXw--#21 z6JvEV*zfSvj(V%pMA{`RbEq_^bmIbLt7Awy_Xl0y1oWbGA5G|!m+h$UvE!lzD%|LDZmo6Vi{MZ za^i^Fi@7pL?dNB>a`KU_c1Jh2hxc4;5rWW_X|U*-IkUx?dRo{9|umNQMVS6S%%6%CnX$p24`AAv<~0U8@_yFnvu0%L%!1U zP+~Pi;7O`%2>VpnQtYyXs?F@D$<%hU=fR0Ws!=~6B128vQ+1<28^hXdnJd77Z;CQT^pwDZx4m9AT!ro8`8thrOIH0l7rnoD7W|g>&Ud z&cCY5Qps;_mOrWviAFmQsEYzM^bIP(4levDbOgUuVr}Ki*a}{m;I@!_VO3i^6KtHj z;L`g40~>(OM#)?>g#jcLl1UB|)X~C4olgo5_;}xu`m^rva=9V1e*ArK+vJhRq=|^X zB>_6UJbVf^nX*{=cNU17F3NA)DR{b91TO79TV^u~I;hHZ4GC`0;n?MDPJ9n_yDW;I zt0CX~vyDaCCk^b)JBizKodU`6z~GPZCx8Kj-|>GX@^y(#BTU;-Tqo(UG3vLJ z0NhU3mtLpo`4gS2ST*Z36jO-8AWr1-^Brt75zQZJ{5uNJ6o>n+%5-j;w!~}wK5~A4iXj#XWT53SujR6rl7_b6? z`hNxWr2ygk)msqmc}srw*WYo~JFFXtWI|_siS@&HO~r@nY|v}p9Ht8PnFdUM;BNs$qSM^1(Rf%7okl-k9jplYNOnwe zUnwpDgLRHCbGbMTw&2^>LBg}&0fE5&$9gqrC=z#BoPwTm5bw^nk*X81$zplgD!Dgx zIi7WY{P~>%5cma%EVLN^Bh+_6ay;bvCDFM6=*6mI#@8_#d#;RF>2{V|wJ{%szbC## zgTl1Gq@C@T^GFJZpFz@wp*L6ThXAs4@7}A2J^vD&?jN21t?j5KYoi{~@knzxvh^#!leGvWaY$u2T6mlKJx2|?@7w{CxEOc`x=*f= z4|7NE58RH`7XXv#+u58K=mu1_hmVCDY7Kybt!$6y4+U$Dx^@Nc4~5G=E~MOVK_IT^ z;OHVXdE5NTsYt+AD2R5=0MlHYX61`irZQT!Orv3&?7WaoW0<%n^W3;&Xe|@2T1HT9 zjE-LovLFcrupkN64}!cqqIdr}v-8d{FV)TcE6n%#;f-9qZuyYu6RJY?BLyWXCBeDM zY3`@d`FV9i1?22z7u$ucZ=>hEIo|a;*wnha>>GY~JPq?UVNO{DyPNCtfIdvWN8@Kf zd%6vU$Hf@I`r>v-Lj%ixwsF@xyn{QT4Z->D&q8DWCm<9!Xa@-EcS+^N{_p=B3E!UY zwXcQC{q#RR`NcyC)L6)>v*;(ch__pJA9difA4mrXsZ|4qq0Htjsvrp@F&akVEf9`Z zB;~1zP|6+?3@DYqrhE`i%lLHkI|-9W+6O2m7^fwUkk#1>$t|~*3p-i&a1a=aFV;*T zOC3ncWTyV6xw6E!JCxi6n2-I`o_S#ra0?V?A~MRjx{$4eaQDvtM^E;*bd-{`GkC|5Ys^H{bJ1rcc5xRegAX;8E2#x94!Snh1 zrp^_w$gGHb$f|&+Ztv&gyOxfjq!JiOQC=`|Qk*4VQrtopX@})5XQ{{kf~xcx`V9St zn*h|KO&NAUQS@L98N(bRV{U#~Rm(UnpI1fE;RpPzmw||G4r)GJu;)9JX ze#^u@gBmxQxar@joq3kB4O2*ZHzbv@)e@m)D^;bFdZxD56z#q~F1Z*zzCO!W$E>1u z=^NBS+fsl2rB`Puw|xG;P^ir*RO0xK!b;=t#4HIt-_VjV9o=0850Q8y+MfYo>+78r zY71h^^B5(&33kLH@h|R-qcl@4stWd3SR*6y=@goaXW#()3`fGDe5XIMf(xefoL*kV zA)N_`=qAdDu}sds`*-in;a^+tqT<)xpK?(H3u%dYh^)4#Pd9WG-hyrExR20TULOT{ zdjCkfFuP6>SW-SG*ZWHHy5XKC*J28_n3u+5H;XwNVZkVpGGkl0{LU#-!baR(XmD`gNbGSwsaTCbKKUEwB9q(mPl!=h*PsPB0XZ3Sn72a%bG4X&kbJ;a6_O` zRAMRDwW;zUYHB=XWUbwRGHRh23`T#bD~eF9dKy*Ta2Z$JK5Xb@X!%l)@r(uSz`&+W z=ptkb6DQAEj#zZoNrl;#b|4*#`G`+Ote3pVD1+@ZoGwK>onHBagR&UfasmSFm~*HB zo@hxokXwfzUO^`FlOL*)n80Wx#UgVc{|c^REXjRoSYY;x{H|8>EC?Cz4;b;V_x}%p z*n;jWdrFX^FX3*{Ie#b`zjP#M4{K`jW>xzI#Y4}OIe&qJ*m+s-)p|6~d>;a%ur)?j zoWu+|Lz7LeAIkZC`WHCw_lvrXo$Uo96W&Es#Lx7_I-r=YpmV|R=DN(oSo)91 zvszktmOmpF>(Z}SQ&m5r;y(hMb;swhyc?Bg(l@F~Y^Qmd>t%`37TV#GYr7rCbxR$@ z_FwHjd7C)%Xko;8hty6b#YnmXzS6w!;5$VJ>zYS^WGs>dlB2of$Cj(JqZ>#6Rk4-!~ z=hf~LZ}Z&082qJI3YG&1@#6|iJa1*9RI#kQ`z3FA@1m&_wlsY;>`Jw2*7D7_tGIt+ zd@N9Z(|dGSvz+PI+Rf-bT2$S#=WK)53)_NUCAMPC9keZ3EjpUmhd5Jf4)YE~XOhKH zh!)HHCGE6@KbBvUwh6JbU+QPsi??dF7oE`3Og;vkl!`_KF8bE<|9wWl{78KR7>$_P zPYcC{r6hVOQ-v?hM^>gK8(CFRrPEQ@LGxEkl=I`zi|u`8?BBE&%V@f7S-Y5{lKR@^ zs&bdS=KM9Ob{DTF=}6lWxsbf!sj>Nu>>AKw#<)&mE>R8|^Vr6Gmqnge4S^}w|Ix6D z$&EWpB1)hYH3s-iXlPI(^%tG9`Rp95zey1Lu_2wPRbP^2KW3s{fLL&IP{1)C{ma5A9#E>?we(vNc2nSD zQG*-W(uplgkJ_F)w|se~5MPd@RDrDI7)_YstbWM4!%!IDPRE*$Q_i|PFpI?;>_?tS ze2$5YqD?m#o()fo-cP#AWbkao&r;&)t}IrN$}>%;Qb|)$oJ)FgSO`we_}QWO()hzS z+B8x~BuU#+YiiWM{Jb8gA7(yYVXKs#v~YHq5hXi@O-EkIK5>sI!`|NtLFFvv6fmrj zW0^}isvs4b}I=Zx^Wz@NQ`T8)eVY6{bBlAp4o>qnO&G3QiR1p&P zEJvBR*I2rntNZ=*@-$s();$(etau=3p+IUR!8*>xF8^sGz(FMqHU6&rX*#|7A*9|$ zpBi&vB_1YoYoXXmTq2?&F}e4Yb0xx|5Xg{zXjn4hHA2a2APC+#94Yq`;~9aFK~hYE6Y2CV%L3eD&{@hORv=k^+&Yz!do zjUHN*E!6P0CN|Fl8o5Im0yqPmoxU=QW%gJ8dE-pW>>tnF)G9GQh zV63OxB3;`3Q?PFF&o*HQwzFNdFKxnbtS6<-L6U)X0lEtrT7=O*A7q97Bn52(bR&QN zO2ru>3=!yR%}O4YEjigSDq2pukIlH%wR3My^GA)S4Ig+-A&|8=*2+h^q4n1^-SRg{ zId!If2Luln5h2xQ3T)DTjMjV?kVS^iPv*|d>UMYY{+#?rVI=Pt(3aE zh9kp`=s!0SsB@DOCCgLka*Jo*aag(U=5XITXGe0l?+$HiwK2XI{m#EuY#gNj zFq;*2C27ISyG>LUW;jV%tshzqUBruLfzcQe#+4jDl#laK7ecBk(bO`q&v4y$@vD5+ zqzquN*SyE}deoBjQMKA#u0e%$PDpE6^(dfm0{UH6o&ac9Ag4*iM+YeEpRUxDVwROf2(aq6mu z+-8C~dP_gQiV$Cf;MRL@HO~dkA%-WEu(bp64Yo_}1!Gyq#E!hjYq~lZ3ylcrx`S9H z3Kvn#gy2(2CJyV6=N#%BIP`6qaAQd&u`5)iJYA>5+Qk`bOq;lz%f9CuSr^fbfvGFp zP^`h^lK#rKe)q)5;rEjr?us5CW3^h=NI};CSt&qU-n3_aWqzXV5FM-!;%|A^Pd488*7hn^q7T%yKg{_$#ox1%>j6qrbju1lz0>~jjc zeKAlWs>Yx_gDb_@s^9w9;HO0+B2*x9Ob??+4hv!+8`5zqN}?&(<=KQM-G9^W!Q(Gt zsUZnMON3JG+4LaIzvT&8IKBH-B@|0gPS!V_(%OIg;CcNFV2|2d#SH3!78l_rFu0Mt zYf~x|UKhvABuKqQr)7jwRp&lPK-%O;4fHhVA6<0T02dZW#tAHF%eOCAJ? zeAC;LTkHBStRH^f1S8JQ_!lc#v(@VeWHRUoT+W#7!qVJItd=8$Dui|5@yMARc(7T4 zKPMTGb%czZ8OWn705ehF%CkY5$>3r_dC0uk9NKW_Kw>lSO@0hB`fNjo$2HzZLVf@g zNpPbvMp#V7h7cUY%$XfJoqfTj{3hONf;^BRhU($|}Q9G*&a?#}Cy@YUIx`ji>P1P6kv? z)Z@_N?Vu4r{G4t#g-%SZ1p5jdeP%apn8%;SHJ`zi+7A+12-U{7;&5M;U&g?xc zR%dQNGz(|>A_Q3BsSAko7%!zaLwzdo{VEU@{9m>-lp0&)3sBQknOcfFae}rNP%m`W zw{OB*W|yBoS?@lKOnRzveyd*~e&O4b?FDe3W04XOKDvI!A*aY?rVBt$ZxZZq6pI(g zRE~t$Moo!y_dIB6*5h%J(Cm?{!bFo7BO%c3mn~qn`c(Cx@ipxsw}ZUtysV0rj-C%s zN7l1_i6#SN@^ly)iO_71NO^B1q0%8$>=Xnu9`s92y%@1eO#{>ZF-9C@l8D>#c%o2# zA#oQaA|vS{Ur$Gi;~WMZ>_l#gS>|ao9Pj>+Gyg_g8b|_PIkIAvnMlyq((A};)^o#qrVyu0#iuIdRzx*RuhnA`?U z*&ulvaT<+m)mm*1`$9zPTypPbtuw*8?bc!bEh+Ze6KZZ%b9myl)xj7i$5t;Pf?!f6 zVAkIApF$mb>XvyNNs#!KU^MoFRC?9AMo8*1*S_-A;&%^2?#e^?0jHACk}@#Bv2RCK zuE%H-L`2;DK2wH2Hed}*X>g_x!AAp#C9n&NKmV6>1o|0F!#mA?b?=#ZOH6E0s`)ej zFGucyTDH9rpdt};Y%9#j5^^0bi4ZXz341mx63&2*DV31!^a~_!#W`)oIBtdiO6Rcg zglo>Vurp#RvH9g+rd@my(?fe@V5G{o3(^W~KFlc+1yVrf|<*lxu zz4A;2FS-`!-3j&0T&wSZmBEXiT0C&_t~N|oyxT{-s;L&9JKrf|ae1 zInBWK2Pzn5vqyY&q_ug!Z?6~ANh96&&gs}2c`RA9=VoeK(h5$Uhh>iUy9X$Wv}E_s z8zJx@*%<_7y0?+_u%4C`mcodgI?d0$VNe>|8*`<=#WSEB44uk$Jmh1N;J zf{u=-RRT;m5F~M?MFq{u%nezZqFn#%+)7z3FrO8ctF7n^D}BWycT+nCBGMbsA}FRE zFfDn}i&DekZ-T(4Op|HsYJk_VI_@r+T28j(oB`Wsc`h(MgGGD94hKS%`ZyaOy5&Hy z-Gsr%@e&*svBn#!=#9XAR|;R>hzn{x+no#U6lvtoDs&au%IQ5k3<;7;w zvCA-7N)lq-=%b$?r#Q{9{yKesjvEzoNE_4+k`6H_WUq~u3^32F~(Pg zRkG3$nNz>>?l}m$+l1AqlguIcZ{>3{Ih7y)W0=IC&IpsR%Q&tl{vk4%wZTEgE+0`- z3prZAho;o8B>a1G>vitb)el#zoPYMMbvCgv1JO)XCVN(>*VFHraW7w)9d%;3x=J66 zoY$E4P>6mBo1xZjl@UmJSFkZq*GSfw44Z`QJ?;(}N)!^PJxZ9ABG45+k@P5{TDMor z2qp1wUk;HB|@R<2s5Y14>$ zNljL%!P55=y|Mj_Z!~7+rN}<<8;{bwe~ivvClT5mz5lMi9)si_I{Oh-xoM`f<<>GN zIl9B(9e#a9EqX`2PEC8@=N=fnA;H~9WtqdK?+I}y{oKtn0-L?VIDLg6d55(74q)T$ z8+-+=D`gWMm}uOWXxx!%+>(9*N&q2Bx<=)RRJ$U9$UV?UZ3k7kc=+~~^YT5Nl7#Ur z@4i+h?zZh^EMA~gp1p8hKW#{mw4Uw=b~pWn)m?n6KD!E8sXmR4eGvpzgM49ZZ6F(; zqV$1JB;aUsjE*}kkGgN}B62KG$1l}xduE3YKA$;&-z{>L%4Eysa|-s=Z~{bUPvVS5 z8A0?)!x%x-Xi?ck4mwHq=9+q>H92-V`@8$4oHP__btUbabTZpXuMxY^gMN(DND(y? zmg+FPFF%qq4rLg+xNsSHe(9bt_>imKHEu8c4IRnGmx~V{yyN}*uhrSUM0FG^1pahR zjo&>BtOrpY%7VfJ0uA7wY^i7|^Lq(gr6&c(jlhGz6I%)*1ns^QPVIAGEWHv1G$p!%kk_3k*^9E2T4{RlIHYKoIDdS5=V&*@oKAHQ zQB$na??dJr;a#bscaDlwm|r>f&G@vW+8lXyy7^P2)2*(XZI`zy zJt!2|)0R|8Io7Yo_K=cl?&Bf_Q4CWkOt9K>3Mv?xu1^J35Mq+hMs|;N28%>uyel`} z-*6)mx8qnvZmzm!Q=mi;0Ya`s)wWX-eP+3BsW`#qU|P+mjUb6Cy$E1#oE}XmJtvZE zxaCjtM&G7xJYHstC9102IIajis^90yL9ebKQ_r#Pq9M*n z7)mgZ9H4G}No%zH0@U~G%EzEA&vf`F1=pyp;zJ-|QzBGv9~P<=K1$Hkvln6tb}EHE zR4e&x5HME&d4x1LPf-H=uuap|A}{7e_n1mN(>IM75#=Nv5`>>f^cg zqa5zc!JnAURdWqi(V-`V^li?LO7IZ*p0dCQcxhloqof-wjSUR{NeJr_cPFZC`DxWI ze5{o!^zw4r6=R{+qBxnw6$FLp=x9q$GQEjV>n#p8vB_|()xn#&8~?DM@np+%&yRoW zczi#9!?FL7i|BKg!EBuZYG!qK3_r7gQw&feqg7IFB_5_EZwb=tw2~rwow0W-X@raOkMXPe(z)jF+d+Pj1?vy;t^<&KLzn8_P{Gto952?0Ak^H5)K8~A=(%C>t% zY3l}vgcR~iuZc}#5^b${aEZ6WKIQ|{;fK0w(;`8y1%VpV8Uzy4)*;mPrbzZShZW|2 z_I$34*e8Gt@2oQo1TJiks1+so+w)hgXOL;vhwW-x^qScn)21Fa_7ql;#Sj*E76vV0 zHB+CucW1^X<7hGV5|hVneWM;-*~HZyPC-&6s63#y*h>ZR_YJQISRzg)c{pB(T5605 zmm_&K_lR$tf57reVU6eVcBktdUBMZdu{p2N&e3Z=029vQh)ylWgp+y&Y7VkK zq$HCUb4W@jH6Mv~KscNiDj&K(6VZN!OYt2n{B3*szs=Q9E6#(3Q7`wQVmya)uj~J| zGWwr48?OI$xHpS!dqSVzrQSLSm)f-x)BZq6?OFY;6aT!9VEc9D|EJym71(DleXM$g ze)y;VKg%8)RK=5Sn(^YR%A!!W?wmN`o*n3)oCuFuQSBy~EmB>ZylbrzYV%?~U)E|sKJWPX({Fz%qc0mP;ICzT^$ta`jr?O92wGF3 z*(lnD|Ks#AXqIbgrkC}b->`lu#pC1pGJyTL4{M~G;cfTxc6D|2{P@z_@qXUFc>eqE zYVoyTduy+kH`{ks_a{x_=;|tJSFH0Hz)It%RPdUe%q;p!v;Rx?-xWx$fk|8JsytOFM`Nc ze0_%bVl5C3zO25qu10gMoYclMh;eTXYTHvzMIH%bW_*$tpLa0*1yMi@ja<1t8d$&J z`mpMy>_m-jLE`Mj1X`;up~WGC&8!42N8-rcmk^b(`_NFVf?jn7bbpf1+?lThuUqZU zDYDO}zqGBEJBl)&KYn54 z62!PKBe#!7av{3WPdIfEq@c^CQrPR-l<9v~l)?6SC}zYFESQ0px*%Qk4xvS>k*_y2 zyh@;X_HRX7fBmWEB^+6vpI5;(OBKapt)kkztZNO|)n#~$%V21~KTOO57~xl%!X38i z)(Y${=@~}B(`>u76KLz!&E}nuK$jZI8QcrGZcXm%p`Cu?XWE+Wo75*OL*F6>j04`0;Ft>LkygSZ!DwY z4^rN;&5O+}7SditKE{ogS8}s`Z*KGLHDz}p5)wzg14v9x{3_ohidltiJ7&hTnYt7DUwJ0x z8`5VPo-t+6ZwYnLI_izZ4F5G3KQiZg>CplWe^GGl56LdHR0htp`Fj)h2j-x5hP zQPte3)lq8(t0`#K533|v+pgfmM5yzCYwK4-evja{OkHPhIvqtr)fwd<)>hN%ZAP~hp%TN*RY-W-O! zZQ3jyS*WEx3Mo}r({d9g&VTdLQe(aM{E5|WYv_J%7e4(N*apW>eg&?PO1 z^PsUP7HyM`H5t%#dp&0K;NIQJwlk)BP-y@DqsU0moQyraiZ7)^1RbbaHni1lABk(H z8Xo3IolC`*uXa-z0iA$mC8Y7E@!t%sNzWF&CaI4F%d*u{KdbU~9paBZ3imA-2+6L? zz0k!Q$6vf~o^Uj9zJhtH-}FqZ{Zrj`!qem8Lv~rqZSj)chd1Ur*J&+z{7svIfOG-B z7g!69Q`y1dYVpGz0)%PB$luLb)`0TEm8JthpXD)DPvC1vbv>OP#?K+~ygYcwZEFIj z8QsXKBs&z)W4md{bq*vP+1RdpEkP<0Cu7CSlWmpZs&6x2UGeT2)qmm5e%FiKaC2!m z?4eN-MJ#X1m!T+C#s%UN{=QN}A^qo@$94WfYpQm|^x#ZhMmn5D<#gY8*pl`MS}|lP zXYu|rC^!A80W_wN7sxd$dK;jXQ!oz|jv>n$h-*`9&chJPzG0RUY4SEkXp~J1>$JXJ z@05}c7#u7<+(F%-&uh)XpHnq8t5 zCOOsQ<2qKe!HvnBFP%lNGb;vd$E}epiD}2J;HD}0;$|=Ixu1Kc+w?M06^CS?7*%vn zd z;vq-cudb6H5gbt8pheO!r0&i`BNc65I4wX7R-_VmZW#))s@;$QaAzTu>$OtEKie}Y zwz>>F_fs&%<05I*O&H=KVr4TSNNv_B&KNXgq6Ylvs6y|_ZRJCUYAvPj_L3MdSnBym zaUl|?r&fd3RR6d2eD$}o=Qbk6l*};u-8}d+&aOCJn*LCK$ zQ#<##pR)RQ9pNuPNKBzhpv~x!8;k=kksW&R=*K3xQ;W_Hpr&n3x>8L3cBHU!l*wSn z5la{Vr2(XR$vl5#F;@+Qi+B;*$(M2kOk4A16kWPDW-^0E-zsAISJerlj-hxw#AJR7 zLs=~Pk8HZ(3GQ*BXbHy+JcLFqjJ$_DCSaG>dIbm`XL*C4Ujmcf z2mZqk&*f&ERDa0M&e|xKh-^#TEsVZ^ac*oAVU<{NaeVFbrqajC@STl@+VP{6U`?CM zWyv)h5J_~A(O~t!Q#r1Et4J(_cCuzhqe7{mV&4A?SN$UN5xJS%!#LR^o!bUBz<$%l z%TU5Zygl$b@9g&iUC&ssL{KxhEIOYWgAsbt)KPPIK0dDNxy`nU_O&XoS?@p#<4e}G zWnZjV2fcW+W=M1Vha)CsB%Pv1QSRC_Hyxn}$;e5;LmtVzL-2DB`^

    3=leig?M4h z@RH)G6!20st(bRBrmqZi}!C(nd6^u+`fNPm6CO~>lY92#mb;3o}@pXam#1@3R zoYx8ST;feBH4WpTe$3rjf-c~$ltpPFi<@8GtYoPJVWC6o)B>Ak;$s0dP5Z$HU)X7Q zn&r+tW|LAe${S7To%AO5r1|q64-UYGjXv{RL^laI%%WtYt!n>l=kRTC&#{V~kmGyi zIZe`L^4X1ny!}DrxNA`2U^IaISb>Ee;^En|ATR#pk9hCayOB)tfIOVxTjz-7O5(#` z1~*Bn#M#fU{E#E{`C9$(&pGvy=`iNe>Lqfz;rt!zt!arK)%&5VgM(vq!FZ_Q)-WYf z6pcVtesiAIQUbl3<$rp|g4%ZlZ6IB)o8M~zZ^omKu(fJxKVOj5}v*fDfo^6(QUk7l@j zV&=*6D;xB%Bg#x~-#M+sJG zfkT*qUB0iyyA+2Ry^{ur*}LeW2p>@0akjj?A!H7aBvqnIIla=*Yk zo%g@`ycijXRb9o2Kqk0XqePV8opJi-l9H&Yaw(n$TaNzI-{0;UT=;t^dsi^9vsbup z7I+nb!#|7Ha$l%2pUekw$^+s?=Cz)I-lW{GfK394B~qu=*s9SVtEJJYisBY{bK%z@y<7^mYHRp)=TQTGtm>O4Tr+eI>e| z^TeQg6hpLwz~UB6PNY2V+-T0Ol|TaSjtY=7!{7rY@))F874WUqJP<=Pz?=bHhsbT|mdlO|yQzsK0HRtUV2G$+yj=ao2}BJ&SVd2pZqqHRx>z%tehupg!4AJ!ko^ zczvDTsM_$;fiviog0(RA3HG&KopV*hqW)|})7*WHoyzg!Dz&KswW;kLzc-%hS+>g- zbN%U4S7C5@wYn{$c;dE0Wjy(LgR|^mF4%CYzFWA!yN%EMCEo3#41ekgO(pNH^H6%t zR-dUlm+$kH?|^}zfxyoj%w5?f76#J2qMe?tsUkWhnYqCd5}L26{LgOF-poA;0a;s~-ACeV*p3xkbVXL$`TFHZRp~mYTd$R)!smWvko^jA_~{boJ0*p-H&Erou~K zb3?**&GL^8sFCbAk5C6B43B7s!8CR+7&p(`2)3-4O%Q>iP*h?Xb2!=Hc()%)$e?M6 zo-1*CFf36ira`N4Oh|!#6i`zB0MI`v-^1)fGUbJYy(!Jqg#e2j)fOR<6;oc^RYMs2(Uh|6flhXtmbWdQFV?8)trzm-%_vJMb{xI{oK2U~nig z!pywj-ZYDyE9aLg)QJ@ZY1vDqO&~!E^n>^JzjeBP&3rbls^@C6YIj@M1-jHpH^JSV zy^%kP&yJ~ANZ-au{42>9l2e});liR7Fotrop^U#I5H_5(%8g-QIX+yct*PyOxHenE zzLzgRQFz$*aU{n`xg7mE6(ES*_AS@9TE3NtU|p_$M;4Y2Zb|z3cJWAtuTKka+V809 zbDAJ~dfjuK%hX zLs|zEK=39((&FMtT#e{2fw})5_UJHDWf_KJ1l#->9M!t_*lCOOcwO32u?7g-sq zwE@&(xKvF&Mgi>NdyFQ=9Dc9aIBqU}boy6L<6P0{+LRXthER^CJszG>8sBl)xH|bo z0GKBnDUi-ets>2P@nldVQM;#ZJg+gcX|+#=j*m6a0$OZ(ww}jgHZt%ej<-iIy7j(x zCmc2P&RmiATkRnt24ES@`=!j%bP{Nqw=D<4hNq<-H^dVl)%QxAt57Z>!0ZSbDpsVI zhvJ{|5T`I*Rhg`ynFKxR*{&1`=h7}o%3O)%dKz7Hqs9=r)NW&$*a_NJqRA3s2jf+$ z=3;!*{e53#iGX5f{p_s*K|d^-D3S9NrCKlbdbgXEE6bRtLRj|+10B;fQL6~@huO#o zdgrX0xq3Kn>(=XX)?+^-&?BwXEs4GHI`kZ5elWy#%8_g}N$;DDSMr5DGwIA`=~4B8 zKozoGy{q^Zp!C0VrA9X=Tystq0l@to#f7mM!R*ZyTx}5f+LU`+_2R%NvTO0Fbsa=> zvOwep95OYU6{9?(4aB$(qz73fu1A904HApXC*tSYP~M_8G< zoV8V(*{TH6DcMer$GEum0?x5rtd?t~=HuB#7^qnnC_82zSs?Tvnft>~CaObNBBl5~ z@jtF7A6z-Kq*fw6{Xbhd} z(Yj8BhrnY)ytD5F$u>yR+HD~&;{OE(=Z@a1FkC~xl9r9ogNsfxlmy$bu@=O82qM%- z$O8xB`dihCU95O=;ep8LTA{(J-PCE~lH^B0vOfEbe$6RSaj`0 z-|1*`@LaHPRv<}%9TYdTN)-yxV}^$otUQMr0MK{M*KaWq!-^`srFGA!;xGW0CnTrnZ!U~wDNuR3#<$H}z?U9|V4_w_41jh~q;cm`rgW9)ny}ML2hlwH zTo@rjwA`TNSCb{CBonDTJE-~tx=e6-W!+Bd;@i_=_PjE;-DfceL=-U)>(wU!mhb<& zg8o2x>+XvKB)lw*0sRY7!R0r2mJN@XXQG|+F2a0 zEYqa0QCu(2@#3b!w!};+M(iRJVgx-kxyeGDNC?(|8}d7b$wpHe|9S}EB$*CJ+zx6f z7H$(mX3lyoU?CV203?&(>aXKQWDLJvT+<1l&CK{p{c77pk>vrTKiO7mA!s=$tpMAo zxl65#i8WYp`c>CKc29^}K1=N^I-xL|_jujY1bK(xzxQIlR`==f1w4nF2zRYz!MLI=b#W z&7lTq4H6kt4iv$wwpIfzNv&wvK3@WExH1wbdk}p*P_AvZBp1jd3WDcUbs9p{4t`6T z%IRg;1!j!UZ&J+xX|OZltEIAld={_3uEqE=I=^2p7666fQe$ldz)g-xeor zS#iT|CI1ke0BeSit@!0B)LGZ}bb6cFqQFqpB6QgL@Ik^zivk%7>=WYWdG!OfLXeS! zZnxZwH`D88D-`D_!RUDl;r93QQpZFKVf>~+qEmqMfqmd@7TY-a{LkYFR?6re#J_i= zOlgxpfw$Yy_F-POOi~^>(lY-b6j#FaOR0ap+V3GqG1%vb5b-jmhY-c~$s!tawnkyR%EF(d+rNe?2@r ze})u-G$}MV+o4l_aWVRZuU>;|X5y-9v`ZGJ_cK|@@2Cr2K1+8IdX4mLAKqAtI8q8th9U}E62DQSV& zqN^jhKlpIQNvO+x7<|2)&6%H}_H9)JDrfwD@p2fT?{(~cw)7@s+1GI6nJ(?f$aS{T zD0uJlAjD>_;cFgQhB6{GdH*_uT;We3ADdGBDeh)^aYIZz-F5aUWWv!dUA%kU7$g(U z^M(ql1I1cuUfT{ub)4hqrg2s!ZT(2q2R|h?HlXfLl>uXWCR7_c%vhlmwTY3#Aa1f; zBtP--o=TukPv2GAbf_G+cu=Zg4rYnhUS$f``}&fe|DFEvr*DAkN#@7FH`Jl>(laHc z@7Kin_g4=krNCEW!$`-o|3m>_n*`WB50oa$0D*%=Kdf4GI~B|t41CS{uZ~wO8gKW* zZ_SCjJDLdFn@%YT-SqKU9)v$Jq_4WijS{3!={8KTqbX*fqQJ4A!>X8+v_!izk`$k8 z!}7K{=4d~HG;My~`H4UoNK%EzXY=tZej>q9_k0%@Q#}O%;TCEjt3N4ut>UHOmZCrz z3;TKk@}5e$(Z)0R4U(=TwvM{;)tUFpFsTX}+}l=SlLzd8PQ4qKAa*aJcpn^WAi&=DGjc zTLB5ss8KC>kVlqF6|P;NAi6N(WJAJ3^GdJIVvTEdG&I0L&!hL9buomL}ncQ&iuMrD_L8^EGgL$$oM=RnfZSulKan^ zTb>HGh5*}uD@VQUj*dLz=r0b>pep`D8VCHh$4$|{q+E%^(Tg6l|74OdMfdfS(yygU zdSkiWE!IY(;4de6A7LSqhr&~$*Wsm?E&6}FB-ut!>n)ICp(sa?`Dqat;_#fqP43J= z&DKQl^raAukDFk zvv-A=$evj(8dWdoCDG7~TK_rE&mG6uTA}kW`{49!B%JWJZq9So0@KrJ$GP7mYVARy7GKng4e!-TK+pItn4-x#KG+lD3+u-1!lBt$>Qo~*!QG$s^ilwIY63A ztN@cUjaMD&xf*6MO7a(axXkE!u7+IERcTs-?3gm`4l5vp1U zB#3bgw!oiCmW=`&VZ39Ug23=JeMbBNi0bv_=nY@tjySxa`0fDZ<=|2?!Pn3_^_(Gk z_)^XDM(qiP z)nNHgdYanYY;XP<2QfRTHO+W9d+ftfxgFooET%wMgWUs9+nZ%0m5K5?ii^&>=;~IO ze+5ZkMJxkI^*Ki2kc!a!Wm6*O&jztQ;1hO(hadTQ?M>GmxJTaXXyVLL!5!!esUyWv zjPNN=3^pO*GgGy4mD&tot!W8;Ol+!^Wh5pSWE|;oN8Wvm5~RfvtAkl+1UENY-wu8| z)WpD;C?g)sfy|ak4t)0^gtqFpamPw}OqP#qRRwin31git(P*})mW&$D>!L>kh z5`Cilc!qjIqw^5#&%NpmGl@1Ghk*vy|IHo^v%u(~pK6i?t{=5IFN`hP zu;%%2x2eNFR>lcO9e6G)QZ67Y880eRB9@oQQFt1Uk$sc77rvbx#937)jFdLW72hkf zRBdx}8B9HTK632*mb^Q!Nq}Zw9aEXvPvoJkA>iS1lXYQnrgjKViKiBbP0qFdapQ2swYkf%v5m$JH*r#S6=5;rtaPqm%x-%W4Fdc1;`QH_Cd z@dpg()EaMFFtu(*f9IWR2L4tahmIPxeLc@}&7=$MEXQ&Z#Bq41_j6vL5asb-oV0sf zbHcZEt(8a$j{&R8KupZW*0us!Xf@jM*8p{1bO%W$ax(z%y(sg;519m#a(8-K={BY_n?)#hP z_G`Rrf@`V%BJ#uEnXn4TIFdMYuW0ful=bwuen&?6%Q*FxGW~|Ns9WaV+ai6Y%~iSh z9lya#OKFs21CrvEYzZ&Hdj~Hkd#XlwwHc@zv}_#(FPEn{g&GZYa4b;$xuQ z(2Bm|1p7RzhEwN8wu5;?Rj8;#``YeW3J;VuT)KZg*^d(!Wu8+>uWnwPqxzIyMc6tb zfVs5h@L>jR&Nt(o;BE72#+VlU1pE8R-9b%xf3m#n-u*=Vx<(IzHxXt;xUDq_eG+IW zbkfudDkPos=yLJX_8~gh47_VNb5!n^lHu}W zf8!~Ptx0LWvsecXfT=(qRN1}FY-q~!yoXy=XlfUI<=lmt@!+Uz&E(}9BT0vMOQA^9 zxDaBZ!7Mnhf5@tvdS)JT6cMB#phH+qXqRi16X4fV#gVFKvpunD62d88bT@erz;$P> zn+s>N+)GZXTTnm?jtRe^BHVk&vF%K6!>|MgLcydg?jl6o~@pH!6y@;=#!>aP9 zcMta~yw%8m^zT3V_aFWHkN*8f|Nf(Y|Ixqy=-+?z??3wYAN~7}{{2V){-b~Y(ZB!b z-+%P)Kl=9{{rivp{YU@)qksRe^sgMm6i7+w(ziONt=@B9UBK)h-zaK$K0FE>UprkC zM(MXEqs1TAMm4D`%1AwXMU?h@ol*w2ueNrD-I1k z7p!+ZRzP5AYd%Rj-mIAmPIg|Ytq|cdYg6W5Yp9Vgs@LN|3PL1Q&T@@1UU?^T0qkZ|+QMBm(a&h`s@A+N zIlf=lE=M8OZ~E5dYxPytX+8`3mB8tIjYHelfAl>k2N+Dosx`r%7rzL`=x6b|P(5%j z(ZPh)-Q8*NC6A?&p}dra(vxh!ZD*LDINanH7dQWg+1TC>Yhcp3X_iK(NIt~Pkf&tsq*UK3z{gZh zCBrlvwx3YOyGwPmLG93mJOa1DYuQr&$&Rb<_HAD_sJOrvUsEQB6 zJU~1MrDS`VM3k>R59F-X9m(?hr^h%YD?^OVsZf3HPWZa-^KD3lhS}p4d%c>)rVU{2 zei%q-kHalqsZ?d1EF;Mv%#NJn$QsXI&sk-5XsJ}x37^D1oeJ1iVrFjuZJ1pi4Ds|H+|% zk~O{dxrySetgf`$D=$}LFwB`c$Ytpb(X(%X;zJK-Ngx=wNhcP!k=>;%h-tdcSlgnrg*E?fYe5lL9fQa8>jci~q7z?Jg03 zHTNiXM;Y0Lun+H80 za13q>UTHP6hC!k{?QMIZ;AQ}?#bvzhXTXO6Skv6OIsL?IFF}|G`o(Q`e33jPL3Nz$ zUUioyCQzp&pVZ1mFrE(uEK2O*q2%kY{OyRsYm&pZlVGBBP`-^>53|APy)RLZ26-5P z?mQ7<*V8sv8(TpRpiSJ;LtPKYB#wjqihS9m0^?Cc4it&*P4J(@Sqckre zMCwQ7pE?1Hl#iD8mflYbKuZhoH8fXv23NkEI*mBFRyG9?xvYKAzE5NLlMT+Uk~18A zz7;puXFrqkJ{@ruMocQ!R0onjjD;td($RQjt=u{V_TaVufR_yuUoWSJ*W*k}w~A&8 zxEi)iG;@cJ>iE@kHj9t^==VT92vTF8&3uW=>OGHC=6;Dh*Ky(c+lQB2HeuL;yT=np zu|y6S;@yFekKn4OHi;~Kf!XTb0TGK>SH8>~u2+Xy&KZ6s$18u(CYVdf14`?|;W@d5 z6LeY}4yV9Jb}KI!kv9C{?QcaV&4*5|2skr?b=Kv7z^*_jyl(g!<$+%zjPSGIMTwoo zL+ej49$Lv+WyCiQ5XGc)fh;g~!RcHFl$ixLY`FMC{;oIQ^z0b9W2||0T`N4Tsr9rSo#w;>knQ1@km$8 zSo`bq&B7{*MjPIaJ;uNbM1@Y2dL0h-Ld&?M|L1zHu5H@ZxC*FqPVSvoP@7yO;Tkz* zIMzY~t%Oz$+L9&d5#NO5ur;MX=1SkP8|@yP2_SmnQ%Pk@4M=QOeFIf#5{IjOO?wDO zh7cH-(i3yGimR023->%j@zQ0!wagECi331L0qA(VJfyvg%mk4nY##lY8@7kz#5S;j z;NW=*HS^gdW%7?j2P%wr{K3$0%ez}<`{Aw-dGgu!hR67`V)89V0PJ`>yIKO% zz>z>YuSFsiKtMGvkB6x}tdu+=+1e^<$Tm-Ng6(2rT1jNbv?G3v+HImT2IgirV| zk3Sg^b`mv>UELWQH9M}Y(LQC}l$(Cx=)s$d>Kds62}A2CUhwYP*an=4VeB>0>B_wT zr`7Na#6$S{+$4HuK7K+^$NG0Ig#+}-mmALW!*DvT)%jgb+~2@$b?p{JDq;c#2r_Qd zB&?)t@a9)Wb;9jh>*Zk#1>C_x%!55BjvqR z`N5mMakIXy@`H1-{u|njr^S`PCFb7K+hnh;4;-VA_}g`(M^igdq2O{^ucGBVI(gF? zzmmc*W7tGr9|%WQEL3@)?B)DQW2Wum(yvUAn8Pm;L)wnQ0ca5XmnTI^0BKalC)j#u zTDbgHinW=r84R~e_Y-rl%(Vs=no3kXJp79zeRS2>c+ijkW+|UNX{X+?_ZEep!wr8p z!(LIv=CpknIs2`l++Q)*Om6;^&5sJ3q%Jk7n-eJqjL~41?kRe2e(sLyX|z7T z2~;R;x6d+|y#Z21&Xt^EgmJ`6Oe1(Rt&L_q+$f7)SZuLlfbOVQ$0G-toU;E0`aC9bY#LCPd>4}W6K#)>n-N~KsN5l8 zUc6FwA)+G~zw}wyT5TMB%_UO;zE79mv~=0=DFoUPP~;IiUEc#Y&?jF_p4>k)bSdtb zK}jmInT5`V&1+$xw#Q+lbY#(ewGG93d~3>>rIM`o~zuf)xL2bMlu$vB`iydUo&wNJf;Qig38* zjUzz!Gs4`_VXX|)4*=JLln8@#GoW33ZGt+12it(W-fuG>wpbl@oN>=k4m|>-qxytC z5W2O+cG4amX`z$k-UNzx-ch?RUeM}jJz5Y8%6-26Y1?ce?B!6iaT!;c8%Jovb|$7c zsun5U_XHH&!3f7Z@udvP*Nm}b;H&!@i9u=@$((z^o%D2Y@OBQn&_>YftR5xi-(})V z-ZCJbv8C&EF1nQiFpJ1JI>2#+40sS{)6Md1ynS2|;gm$O@XJxd5;ph;^Sn^&9hwv1 z{pyc!30WG~j#A$@FKKq6p*ff-(y&u%b_lECpMaX{;jj-ohHJ(oh+0y5KwIp^@G~(* z*%XcNdTBVc$NPH<2#l&ZT8qOeX*1v@k#T#OIOGteeo%{r@738rRE-N?W`Tt(viV@V z62pYLE86t^!fwKz3y>@`|3qWP1|YTTB5qi^w7A*`QH`X*5Yvs)$M@sLl{HsU#L~_uG=Mz4UFrjQ8|4$Y{?`hc`H{lZDHw z>o_i(a{hd9MTp_#t+C`Y9FXEqgtK+Cy|tWY8jUgvy5^a+Ze|~VBW`WEtxI+7U5~N+ z6DXw^Yjr2$H**xu>%Jc^%0gq*ANy~8InoE^Aan2wY#PI`0+BpnAX^K?T$JMgenSwE zu-Pn(oYeiUW&hg%DUrR~DJ;FLZAppsyzq%L{5sn>NQ15zwhXi;YlA5c+};Jl_ z_|{|D;$mtrJX5|-;S5P}J5`su+nwH|U1vxmW`c7BmKb_oiy3uo8-XL%;Ne%0isjZu> zyexEc2tb3u?h&krFHvKk`OwSNkHB{1aQzOD_;*A&DP}3&)!!~w8`il3*-m)o>8|4Y z#+v9^jQiwk?%V8-f@NVIl_PNZfzH?rrFas^cyp65@h4!R#78)gFP-5?qn8ZaiDLR{ ztM)ti?9w)`({w=c9ArN~6J1j>Du|SgnQm)xfx4H3S?wXyf;#Wv4*MXv4JpX`0A`wg z+LElVt;x-}%N@7Lw!I!#DCFO3VC)+rvJKTmspnj2ZA*Ly+(T>bGK^LEIV)>hxnh=wpw;pf1N(?3eYFKjRK$P z>Ca|8n1dtx|Z|8(!fm1{`!izYY1vTAAd=^5ZyVUM98Qv18gn@GO2e)kQHgmsa4a_D8=; zrx320;=W%U`rIzJw&6RDeG(Dn1jOmj%;j2n^9QIS56YOWWzS`{8=jkw6%$x7Co5x4 zTLoyNq%5`{8J4o&1nK1qAs9eLdgE%^bN{B26hsCEts`5pq8o5KF4;!#Ueo+s9l z^w&W4RHDYQ)2m;jz&CEkDZfdoZUDsqY`c|UET#^f{uGFQj*FC3BuXKt3^L2xA|Xwh zzXxdKwr7%baB%H-e~EV3OYM*Q6{8TkY*U0HiU^H!*2@UA)51g6v`1ZEIKVVYc@h0i zij9(g++v)qFK&5GVeas-my7?vcd>xT$>57wq8v#s6Q9Y`4|yoOEMVEn1`Kf*_&{Y%VDge=((Vh2cF~tt6=bAQ z;NrJvNuc>ZIBA}ea{f340!|bMP#2mM1TINRON#ORGzSbH{Pq66-@$1vf?FUE=Sh%# zS1eTAr!b^RL%E#Y-$QWutCZT^a56ha5eY9et5p7GDMAU_^z%ABeJ=Ojm*w<}yveP< z0QgzYfUw$$?VP7l^Vx?TL{DgS%b15v?^lO_HV0TE50! zVO_cMFe*h_I3~RyJ#*sebl&S@2KDcYGnde6?KaZO?rV?J$h@l?Dvv3*_`zb|o;3_p znau5*`Jr>H3DoF3?QL}ljh#;WL#r=Isa!wSdfMlTCJNd(0#&Eo=(MP-cE$>GW2(%H zwr1DOz>9t+#>LU!bebbyhx>Yo9x+cYWI5?5(dJ?a@fmAY8rqh$cgzG7tjx|i2B||+ zo83yC>Nb!OS;qI9glM>@i@LCPu`B8nQcL~_^3LY(RLJ~nDnA33(KPt zJU1+`RaXD3X*z9113`$HpPrOVPY|9}1ON$kx2Ts=GR^PG;%m-fu3=d?q8otL_+Ft) zhmBA~2G4pT42z4>6y_$}EuLwa!VNMdB{%HeAr59oygrkT1J2}BwrFaB^snD+rLqg~ zJ^HMDGEHb)4lval5cIc4FOO0%1+hk$K=N`&TTP672r}}|GjxXNm?mxy{%Ap2Q@Jm| z-bFxKwpgTI#*5@MjWZb4;IddAcx663elnT=^^~**~MPtgXX$ zGB2Yy&=33%T#~J#P2ltXXrzL%Kp8O$cjCBqq)`GvYN>jiA|J;J6b(pU!Z!c$w?0GI zvzQgLQ~Qpv^bqfE3ikNy+U2VvU3Y;vRUmB9gR8Kg89NnItqHt_NH<3?J;~R(nv|%S zVZ;p^67@SV?C@RGj`28Gw7YEaHS1!J1zd57GNM050s0KC-9-bDlw*xg$;72*F|aIz z&|bFix2D0x{q5@<)NY=yD)fJbtji#CR3P3l0C)2|x*&Nt#4xubq5K>*E>TUrH;$e` z$`g*Tmw&fpR-m<5Ur2NlzK)q$~1xr<9q{qLlIN4j%_v+oJcocv}kI z*ZpB^IB(Ol+y)<2yAqqkkto+)iLl+rtuaeqxO%jRsJ`x=Ev$IvC>OAj`G@ajC%V`e zj}rT7iXj8wQQ?wnyUq}xN(ZU3X=k7g_7D*It=zz((tyr58p2L%R(NQ{XPO|p|LWrP zyq2QeyI6ggu~0h&CqL@5Oz&BJX`jB&;oM0i?Ddn198+^HtJmjk3E2@IFdZVC)%cy$ zmTvg(`@6xJXS1(Zm`OG0mgZ^zZT zOu_7?ww#g^@P6PG6T{nvCht(j{Yxsw+-(f9^F)5PW$sL|!$wjklLU%CS^So1G_Ogt zyOOFMyjZK&b%#(gJdv4a1vqJX?q7TdXOHugtG`$@hNDgwE%nXcU-zJjX$5 z8D1FFmo20*BBLByo)K)5cXw+b>ObDU$3u36Yn?D}h;ebbxu+cW{#6CGXL|ll%?K&e zIs&1G47DQ4yB=kMYCu5EIvQqUemb!% z56vA;kN)AGh{LZ<@ZawLNj(eOYoVn4dk)npD4qp{8h>=Tyv;;xfQ4Eu_*CtazapdR z%w~a-5JqQ0dqKdE@OrHqF;j@$5>fO-NKkdq!C9q?g^|qY7&ouBjp$`2z0B4IFsQ>n zlG9ydf0=f#%$HaVDRF`{l^R3G$ONicGoU})5{Ez6SHp_1Hb;)gSbu5j{d9@;zsFXw zp_Zb`wJDjAJ>RoM``C##?SoI4YtzNsVDhoGpOL*;--Zp~c)ju@ohVDuea@Gs+@5$% z@^*MBUEFna@ExNqZk7Ty;|e{1NF(X*a7DQy1q~`356v0%l!V`kko&54*+NK-H{T&N zN(1tKerHuy8YOT}F3eEVGOxuc5HLUDBJ?KW$&jJ6Z->hz{X|5e z(d=L^2xl!C!{2H@a9T{+1t+bU0Ol92M|{8T=Ru-9p}NA#wIm;sx^;*wmG9VNjk^F8 zd9K*yv$w1XI4(fz>FBn5<(byPad@K9hSB^ai6}~hIe^&_6)~((pz~9mq_c+)7Q4To_hku9cRMcbG0I52>@RqghvL8eYYfG#;^Hdnq5~sNj0X+s@%8>BZZB2C0xC9B49TMI)Yf2q)S*Kr!P}tV$3;aYSK}VC{)^is+ypCV z^y>t8&OuJEc(Wix-rx;ivhOqAR^u_ge6J8GR7XuvaAvZe7ht&HZsHN~zJB!J@sCB% zGcan#f2%fp%Sf+abWm~6va1D_&&TwTD?31x<96bepJVW1!y$A8e9U>aV|Lv{LZRp~ z#HL_KKxbnXhfOo*keU~Nky4|6JZb_b4ktb_sXu*c-YglAYWu6hG>8MX8W$#v`H$|N znsl0z?kTCgmq@s*NK*yok&#>FY)HF^*4Z7h1W%}y2EGNlwOK`S%7tYu`Tm_#F?-E; zt;%;~$2q)KE`3Iqp*Fgkb5~;AwD4Yh^{AQHR1>%gO!=A=$5Bp6d2ExebS&HSM?4u- zPo6d#k0-JiF$IA`ZNJUd>EgH#FiK;|Z~0BwRu{;%pMUY+FYdfQv+V;~ec#*l)h|(@ zEhP|M*rf`eGTL9!0+fyO)@#+vSUQ=oa@z{H_2Q)+h<#FE>ck=VhG4psdUNachan}G zp#~OLzZ)D0(F07c-I~Djq!KZ7gGr_HZ`{W?nig|;`;y^_#f6@nUE|C%!Cceiw{-EO zD(U{`I!?@yLu~dJ}6RI96<573w zC>A3__a3jbDb8{V4aH$0j8B}VPTv@(j!ynXR9k5L{r$V{`DyFVhDP?y$5sxA1mCgb znOYKna*OYudGR|{^FmeBr3F3t+Ey#2a9k-(qXw8rY&>no=haMK7Z6mvBv2*hNE(Kt zoK86ja9lrQPmm8B*e7rWJ>&@kJ3)QYS%L!8BE$r;u5YkY5i*IG z-;o4ycf9=w1Sw~{EK}|>;n#nghshaYWVHAtwf~Xyki=7wqEHq5Q*Vya#Rg7thDxmr z6f}tn9to%LLb9j??KQ#&L9yn=Q-i&r5#+;xu|_?iA#(&XsC^Tx78cnH?8O0Vg}DM_ z{8$2NRh2mwnN`n<4UCOP4(VD4YJv$v1s<&chNdR7bKk9o=&cm?G6g!LK^UbG^g=zN zK^UYN)WypzU^Z>j_*T8&c{7L$~8Hw%ob!nmYVaOY1Y|91$lDAEiOQ%mdr_B-Xh80hVd!h$l2@` zLzM$tln~4oL*?^pl<4&qL#2bp%0Yq?f7scm{{=JQM#5U~_e&KIU>drMumCEC(4!Wg zi3HWR5}_6QZPUYaqtuCp2HV~V#&sis%r*e@dtg=q`QW`TF2pc~MAE0elDL`82n}l< zLr~Jt@x~3Q@Fmcf_hn9Fmy`z%If4w%11)2S0N$DzVtCY_GCWr%07)EoR#Z{NHINYp zR+kN=nTnv^Vv`AlX(1|@7ANWtGe$8LMt&(c58BqAkXyd&Xr4v-yFUdEU+q7lu>8Hh z;zllUNN90^eaJm@p@x6kufvPEfP@wiY0+ex5PeE9m8jD9uo!D$<-j##Z+hJ~RmV{`wSiS|R;fPf7-F+&kAf+4g9ypePXuw z8!cpa4aA4C(kv=qo4J>pn4=l?V!Bia*IjQ!=u-41Im^)HDfti@_ZR<61C?g}bnS6g z>2)UcoZ#)ZJbhF=g{my8EXFpQD%W`))PB8pIu6Lfv74+=n z`pozkI!)UWvNiW5@5ghVa&D#Dh zA~~N_EZUbFd+&6$lK@US>2+XUIiJi|@27DyY}w_}7)^OqrN$sn5bB1wu_5vfR0(|V zJp|O}llzJM{z~At+DEgr1(u@_6L;jev^-8pG(W2iQ4_^N-sGUUbVrz}P`BaGxt*VV zB0;BB=PkiD!zrccMyDPawv;_^H;?RpvnU~+RmgU18&owJnakeqD{XSu1*vF~??7Q1 z!UfTLHwiDez4cYfn?8|b)WHD_SBMk$nDHu%8{Gb4M%g?3RRKoTc3uLBta z>EYf?jtht3Hqc@Hz=48ZhvZ}x*X7gcr4~lvC&rUXV0lp7dW@TEAp>U;p&~Tsl0A!G zskE%G26xNRpOqR)lo6}xwY^tp&0p@MOHq8*XDJk}&N}o`=_}ZeVJWyBA3z?KDoAIo zK#PifB{_iIQ#qj$;3DPt^1`t!K-$vJmOU^3a&xD9_DQ5Ej0;S6U))E9i%@miyF61? zPnuY&>5Hhr2OW++l{tRS!SK!)&$x)s!Kpu6O)Pb9Ft2HM91TMeQOu^H)u&QSY{u!A zA-0*;N&cYw`eyZ6tl}|J>aj5=U5W=1wO*(a%UJvlJ;|bGmi{wQtzQ5P_ zJk4?#sKOc!VXsPbe7GmU@d6+!?BMY3`G2q-<{Hb>7LpUOcsuFB!}T_4SCp< zQSjMFhPm`kdojVR0ITKdAn@*@m0mcM{`n8+T80H;vP}{oq7vchXGwM3ZjGguL1Q-n zv5532CTt>ZrNvE&^?yF1vaIbf!uvG}l1?wSv;yk)O{ebU&Wf*Z#{=JLv1osHdaFZ8gGF?$|laSi>+V`2Cg+bgUCeB z89C@GlEdJ?kFbxItl?B`6QvY058vZHG@puklRGY>4!jud*DsHmwIrPkpR z1nSS7TAf?McI5F307akE(cDL#{N?5oW;X0`RIIjm%q&GU%h$uYohlN!O^TR`7!(`A$|_+ zjQT{sU^wcS+VA2xVp>k=em~N5(mv|UW32RH2^i;ui$4VU_HTR)`gJ-8isDxB*;hR` zDQ!BPd-V)58)7N(B61v&3q47Y9v={qoCqjz-gf1faT{}H-BPQ2QIq2DXf3!`N%hCs zVvA?8ND^MJfK*naYh?l$#r1f)O~&0%CFE(4PbXBOw}CiWQS4NzfY%W)K6z(^Wh6Fa zKkq_m{IK(v>sb>h{ayQ>XiP)<)kUCpiQW?z}esjwa5D|h|0$GT%dPBCbT#12$T7qf3( zF?&sO3Pu~Pzmx$~0SAN2lkDGC*b{DWkPT!ce>VbRTNQJ8h2itYox6=xX(KM-0Po16 zWfw%GPj-$OgGhY`_Cz=>-S2hgzlIb44ow-h3k0uBVvC_>-OqJX9M!VJGdJ&&t1<>4 zmHQe@gQ>!gB=}$?SglSsrN`q!p&_LW+l08Kv(oYcwDEqK((4)grq2Q6Ll_haml4grxk#bnMEX-J9v(RqKhAtdu(`u`9CZlq zi-$5UAZ`}&3^5aft_wgAei!Y`L)vzWtE(55%&iAjCh-!(q>!dI6{A@Gj31PSv>z*0 zp>1zzAHTXxOvUMxng&_{cPm(Y_XH;#C2&%fu^Jn_Wo|rq!Mb_JmSw)y;!I$eJUtiD zD2ew=(X=D-ddhIE@ze-4f1-)Q#?&$R%Xqx?uqIqMGGA27RT$Vji*bre7839tRRrmv zqdI3&*1&&@X^{SzDJbh|D&=LA1Xi1!)H#avikNZZJACArtl}$9U-bKvP-~0h^h;o} zMBte|Cw4W|4u5^-vcJyWTAJUW-BVHw9;(q@O3yv+bB-oJjNCH0>8~VxBM&NW*pZyH z_;$E8JG=An^eFm4IQvd-+ml-7TW@?jPGWK(U?`Aw5#o=neNe40Jzkz!7*0^GApymM z8$#p;zBXrD9Znb2xhhI2%bfTfopCu7Nl`3)LRQKdXbxE`p|K1_)!4qzH%Z zQ(du2D&86mc%&!gS_{x%OeSy_6j1|pY#sFinR-1(1smoXwLU}-PE(m{gcR#at=rArhh-mul@*INSfy~w(<*{)>ZI#-q zlNj6Q9uKo$Aw7NNu9PJ=_H9_?XPVLgUgd_`V!G*%ZiHXj@C?o6KN7Ohrd5Yl^s1a7 z5y>?m0z25M;imU_>v&Ca=f06Qrbv6Nt&CiN&5Y*8EiFV?q?f%Gm>$eCCsgPFqaCVI z99;Sls^y!Q2mtnkbDc@%p%ebNXqZyVM@rMEtci%9Q?c|_g~l_m_J=-2h6W>cl2_i; z^24#nSB3{r6Cg?9Zl5N(z{E)js|!nuV@26%bwzE08WybpTTH5j>UGGk{%w(pKMOud2useg|UQKw_NYEin>% z&0S#{R{I6vd(SkQ@!q=`2^IMZZu-a~NH#DVyO0M#n{}HGFSx`Vi)@M+E1eXeaNC*| zDm;njpHx;yK>F76)<&~)4@dSeAWF}L2=QA06E)?`q)y~J^Y3aeVUnu~JA(-gEJYyjcIsWnjAwlnc+ z$Fy{tY4;BFY7Y~2fEKz+l7=IIOhD~WMvMYz>ad|AADUo0g4%F1B^M;GGsvs)my-nz z1MM=f)wMx$p%_Eep7?f}uczZxZ!m{ydV2^j^DB|IHR9oUA4IVuC+5S<+NDkhQ#Jxe zd&EH#J_uW!N)2!ftNL(##~z}1)bSgep?dD4ZoH0`8?4GG^DCYQA0a?aHN|L05bR#e7sa62q?BO0e$ny65^dZkqNrn;{!-wUWFpr{X< zndWU%g4qh_;cgUobIvU?F{7EO88dt~-E7=zTdx0XEC{Uz`1U9q?0f<79B5Hc+wdn= z3X=`j5bfOPp({zoz|l@~@`mRx@%|Qo$r-$zd^Y(>MQ_WM@4yqu%{=N`4-ssCx30{0#p-)Bk=( zekMN?az(C)`>*e{fi9eq2e7QJ=p9;bFCc+PHTLAQ$zJH$o=hH9-@{?v8SCi&HiIJR?^kwf1ikIPSrJBgy-tHotg( z1Rfo$&}hK!Cac<4+uCf4f|Y6#5eYj*32jTR+!~+GI6z`E)csh1GpgJe5x+AI7w(cm z%&m6miQM5YQ*xn!jKLGjOa?FI!s6Dvv&D; zQc3KV&3Sfa63C3BWX6^h%FaWHHFsa=A@?j(b=J688KCFrZ7SQ-^h9qExH077gm!+{ z<&V~!)smY)iuR>>dk^!TL;Km8GYYiho?b;dVmx&Od>t`FtNLmL0}%45wk?OmOJ zIB9x&3XuVZNer8%Eb~ypo#PpvS^-7!ss;vfMsoH5r=k~NaA44yNN0PPo+-J3){>cO zgu_fvs);q4+-E1BkqLQns#bZ9+XyDn6-{)&w@(ePmP#p5Ga{jg0Fdtp4{gQHQQ(FT z$qmKM&Pk6_*HYem+jCo`Za-&oE>dnoA*Brc)}fZE#6IcM}zJdg*5!wySt zu>V)!b9NhXuKuuV3(hHf)M=^J962G8T*5SeU0|j+E!lH2A%7FOD4(O$n*(&UJ<0}~ z;k8V;fa#rsSwhS^SSOxuOWoQwvP`!({L=TsXCnAA>_O`uH*m&qG@!PFlhm&;Y#xDH zt$=A~KTR@r&oa9aOsxoKk^45WT49%}o05whmzZyx%`>_f+$%u~hZ@-%2yz&^3EZcA zTL4}$%m*-N%e&lrm$h&4ILjSZj^gg3uJqf^0I_GG4@C79OYugpArcKO8#XyC{L;kb zoVC{Jur;E+LCiozEd61U@AGY!`S7+u7?^F=`LHPpqpm*cIyY$Z8pkOB!@ANVJT#l(-k`!Rv2Ma)yl)2XsnL91hq3dd3pv(5X}pH8pDx&vfUyj zP9B}%TYu6>XUmqknUt+_uaO&qohImzZ{7n%q)A zmxJ@n2Wh_G_Wr88(dA2NLhSg!nS8z0%)$XkxubmZxZv!bRR9p(RqPFodV;|QEwCYf`TB*iJ3V@XYE&~MCPYHQb%L;@FO_CfxA4Jg$ie%WSQWSInt2L@` z-G`b`e-y}8dN-5+LkD;H;)P6Dl#-&=0(FK;2McJvOxJ{#qRw>ygh!0mIZbam4Snli z{%XD0CR8ci*cUVKbo%tk*2A28paDARyIcctn>-6f-s;?bv&Oe>g_0V+&a*gW`i;)y zP!COv%vF^&JHxLYN?x&uw6&8t%_J5AZpylzw(}6-7Js*9LS$?}Qk3;<>wT#=AS}XW zTiPXfJS_OMwr*>Sx?MXuA}4s~u_UDs)#=vLr@Q&;LlKww15mn(=@&759+q*MvGDZy z3+?oie8n?bW^12(ca1#(DT(Fxyc8g_lY3gCylJatO7gEK-(I}?;qAqnSEq`jc!Y4> z+a?d)ZMpNB?gR}l-Oj|BcB>fx-emfrINhpN!f3W8>;ss`?zi2i0U^ztLcPB#KC?&3 z3#(8@NkCD;k0#hJs7K2^U6IEXaQf&Pk;lCK7Z6_BDK~1CQm0BA)rbv{UDt05P+f#t zpy-NFy1oO+ksCX}HgXMpkgqZ-E1rCjiR4c8J<>4K_hhT5Il1`mn!M!lE?hYiD`vJ) zDyS(lrj#HM4-BrRjV@sy3L*U_S2H%-)@8NDSRW+A?#@(qq2A??A{LXLHkUU6h{^VF zAcPil-fUIVA)iLd*n21$TiWPV8S74L)aId1$b->8) z(+bGO8nus@W$Jv<5T?=@~QuSBgK4tKCm3$e-f}O<#VUVam3=*MDkd4J*o-oh|WvzYAb5SPDz#PLZ-qDe)st^FNWybiZ z44y=3Cd5inmoMYT$vlDRp8Us}k;|8c!8-uUDVKNNfwDE0IiAEP7c2Uc$jPfOu646c zf1yG1a-L9n<;0GuSHkw>s?ne7k}0_)O3<-)cZD1EJ$XqhiZTvxAfFZ59*9bOnF0I) zJmLr`JIE6Wu>Hjwk&X~5oQDj{{+nvy3;Z7H;XfyDMb7*w-0v7oiQ2hm`0=}vSIotqz$7OlRIYz=)}uDRqNBi~NU0CT6A>);@ai&AZD{o%rV6rRjtEDCQf@ zhbH*+7N@gh*y6qdYNU=V?CE5XqOP+bUl$EyApIeq_D`t6i?6~ncK(cJ>?ml@r+k%B_EtljdX}O%%I!@2;Uun$vikH6zwR#gwJBbalJQu^X_t|vs!VI__JEw-FsHU57(@` z=Lria4B8WSxZ`^J#gzV97n2g*laf`VPW0vxa~@i>WBjR&N#yM`t^#e|BNv(%0c|zM zzF@0J76F`QT>s1yrhCW`;cMdASdmQ%Z{TKthi@@$+;$TutXR(DGU?QD>OV@WbTXCBBssnNH&EWRns$y8aQ^Pq#mhIZrmOU~OYtzN#H5yN!Ykj&7y-2t z3H}lTP*Zpud8EhOR4vsIVthHxo|A7Rn)AU9W?Yy;RDSKHq~Ia2#Y^XPmf=`eC1Yo< zrRn_ROA(FXq}A1cOc7Mqfa1Q70rGJc<(SjPCp&D8b@NPnw69y=hqOmnn3aHtaX`A7 zi&Cx4vb~mAmB1jJ*OHY}eD$X?xS@j%8vBlU%bNBL?K?K_8(ICe_5Au74oYCOu9g^y z50+Og4)hK+0+#*qV;hTG=!+QXA6?hdp+~VAZQT?1N;qZ@=MSR1N-#3Pk3#riwo%S+AYunI${A1G zDAQc1u}Cb&<-7B|k&ht#eUf^A)VaCCaK_1a+sh#hUoHe>b82kcpeUxx`DwcYy9x{gS)!7xvp? zaN!$T#95Ne-5puLevJ@T7|dFT0)Qx}inb1K6al|8f2G+XfV+{Um6Ob~sxh!v1Fd z@$=88htEjH?--fOzm}pd8YE&2hxO9buyojnW-UrqeZ)AVpnbBnm*DP> zM5yzm8;sq8^3n%KOLQy;gnnQKN&8?;SV~@4He-U6i&9|KProBMHx=`=vLdz6ZBpJH zSAhW}P(nxu&{{vDi$!f#RLQths}qpXwrp_|m#tR5#I;oC8 zn|mz}u?if^JqhI^z|4M&lKCQsWi!@Zr_2#jD+X*nQ1X zku%Qiphb)bl!GiBCNd>&G*iaSMQ2iEb%n%Z7Q>r1Pj93?Q#b;cwY?3vcm&x7{Q;SB zh~?|6JcP4=7GOt)UrwKcCEexz!vK1>m zaHl31(&W4?X`X73!Gx8-NZ5ypm1-`UV*iqyeBnz{*Pv^Ip8qKG1iUFeP#JDFtl2F> z-!vP0iCqk-sMf+VRqGAXF@Y7XMO|v2mnkm-1p5l1ASe+cWalv5 z?Lwm)m&kk4BPk1=!zt$u%`MO zz#a=TA`rTPQiZKSH`<9J9dL4y8v;;B-Mptk2JpAGWO@VmmffSX~T?Ob9sNdIZ5y&-4bq6*!(F2?~+bqIDZ8z>FV{C*1 zkO%Nf4?+txSR}Pl#E>7WIcwq-!lgSSmTp9X1hjKou{ISC`2#J}i>tx>v9?Io-0upd z=4+z9r!xRn#vBHZ!)wz!_Dg!f`V4H9P^@gjljU6!yM+OPBph4dW+u}z$w3#Zmi$pWy9%EyzQoaIg)rtk`a2-HnZr+_6B6vUvYzs z@_+!XfCAIg=i?@Y1mR%OIp}yG3lT8u;RV7D)Iz~J3}?0wX>PKiK(Z`o!o0X!&1DbP z2=j9MKoO{;Zp71=y9ba#_4vqS$yO9TeyPT7086Xy;*uy~u3`vg!pfB-Vt&?&%R9P% z8qnR%HL5bTUNAnGyY{sSdS^}LYOypGVk1)Fo2zla!nVedJwSKk4Aca~t9kS+Mk*qB zZKsalwT&AwAb+?rM%4Ch+=pk@65DnkwnU4VT^JTVx&lTF)hH39EoMZJjd343&cPVx zQH1-@ku0K|voY>NN4|(+6HyXKOUj5K`>S>zI)~_oodq zui9$blNhsC%a;4__;@-!#=wt8VW_M5|(7xVUU{L8+O#ELmP<>jTPMcH~0I?9J7ll*U~K=2Hj4-h!<~g@q8u zl?l(X{GQy?Qigj>#*0}gzvog!Ef;knuD_00tOOzSM=-heGtrIjOu@Caa5~tk$eBY$ z3EP(j*=+VkN9KL%hsH5iL92?D`E&ALC+~m%^W^j|Cnw)Onf&|Te}D3R3Lk!V`j^w6 z%^$x%Jv}-3{;M~Cy}5ezcYgZw_j$d#!@qt$`JTP{dnA$5)4%*{RP$OAAJ40aC?`1d z&jG70f`}QQZT3rZNLeOkrSt zaW#B7P3}-jvcowAxyi}1Jv!2U@D-yWTOgXEArL~OOpUi8gl%e>f6;{+u~(boQlcbp zKukFJ^pR)~0R9tutz;apGb(gS#-EMRCt@ez&ot9Ss#F5>ObsV9hq_r=1Y90zOEIu` zUYWgW9D!;HsW+nZ2WHVs`VHQ+C^DV^^gU>*FwJ(^+yJKU2dE=oOSQ;FuJ7?u)jP&E z@WG(=12~E9-DzH1`$teOl&&_f z>nh?Kyq(WPzHsrjnjNZ+gB%Kc-t)Ak86E`K_(75{t}l@+Uo5N10|U=4p6Ly*F^%Ni z0M+4LCgGr|w4D#F@lX_#t^qes$3wP4Oah9vUXgt(0J?c$D<&#o5{%+%YNfAe>t-LJ zykr9uZokGO5i1WXJzz+0kCET+oAqJSU?U2`@j(BS)Q%Qipz@Lu_?J$0*4#AfHRpB!02Zwsq>2a zqmR^+_Wdvnu8?v$C$dh|p;(k`V#J});_qXq8Z@X7I*MJrX-P4%t4M>L$D+@<-jdx| zZs@znlSUMRi46eSyafp;TkEJHv=(T;PR^ zNv;!^DN1IgxMR6ZO;D%7jZX+9CMp1D#ze3}=!PK4lI9E6q;jul3Txyms{w?p8D=)< zTuIoxf@zs4fy8>e-scw66ur#;y{d_ED_$~#Yk)<>VW8_A7j}Ed!-i?avlpzAO>icj zXNoau=|x*^;IM;f2vVimSkbQ-;3P_+&n0#GhB`mSMThm^-iS2LpVukf);!!KUtn^N z1oIZ&5ao7Q0hOv-&6X_b{q~mTMtYstN;A-;3A}FdG{z+o5m9Kak)F1gd{uKP9(5h< z7V04X*k?7X*Kr#*GAN=vmAM#KEa*A;T}!~x!m!ooF(K!~sQ6PoAqG3DeS3_IF#P8xwh1}U$yB+QEkrN7JWQ?Tg0a}kTr_1mh2t8NS)5s#KXFFs<;q_ zYNJ&y5>9VVA|g7*#&kd=EyRPx-QD3v7gv|?u+a+_@P|3=qn*U&nlZ%=8pI?RX!jkX zL-g=!@8PeYZijZf*<$P^Q*VkU2|b-cnLwJab&y`Q_zvpndL+EGA2jUT&-TZr8kVlXZSr_aN>TPF)4_j#6i;`UA z6P>JYoi|$UHW9>VQ=oaAWT%BTXst{B;_6Z}>62XK$oBBwIKd4qX}!t~lCjc<$tRCJ zVI5c|BOdWQRp$X5i(tr*b&{~it)$I{ZYAXA2++f`qE)hFl1LG?+zH{itZ0@swYv2J z5Fl}CUy>_`#<6fG`(9f>i;QX$)x6FV40YtydQ6bshN1vg4T)AdK1WYNV_k@2sb(R8 zWp0h9=+ac41w~n-34ST!A{u8jxqF~xDgoetRvas~ouSm`3RwB^aoiSdrkUqmIn(X* zN%LFWHtou&r&jir%+>&F1|g1UU5OQ~cmftcK6kupD_f<)#8Sj?ToEm2g;%3Q?6G9217`|lU}mO&AC!a9fdNubAt(CU8YWhHzqPp zO86c54t(eiB{Z*QnRt+-dJxv?h0w4l4l)tK+{#?rJBklR7c!W1afC;uRpLX zQ~&xncEI%Q={;ZMnBC8%F-_{^nv=8?MG-3~skMIE?LCBnl=nB!t*x;2K)A~Qoy=xRM{;Us+BF3*C#n@-HgDu~0Wj1l`9_z3x95w^ z%O0=~#0f!YLqPigdTz9dU`kFdvSR7HIr3n!yve4nwnxQncNE1JrI7NiAvkUlaib@A6fMiX{I2$P0(9$UI z&Ih`l=B(0qRK_Rp&b;@pB@KX(yO&nWoQ-F=fFCwK)$^p+F{k_cDO(*-$U0npfAilz<>ElPIJMIAQ|I*p;OP{NRv*43AMVJo|uP-O_cDbsju7({zm0hAjD1lQY`a9+FYX2Jd-|8SVJx zOsj<^vxxfZ;dd`F2OC~pF(+kS^Muh`qJ_G6zM?r@u!$u*X_AAy)tB9Cq_cw9RQ$*N z@z1+I-WiyO&JMwoIev0&Ug72rX@TUL_l=Rs#fn!IGhxnlXFz;P8i}vxUYaL>0|n!n z*#}!|(@_9G(|+KdER61@MD3jLRe_}tU~f-!4xVuwj2?jmA~gy7kT90&#{c@*&8XWE z0w4~PqP3pQgBqES&R9l%tXa7xN-i{6A+a$$iO3~Sb?%Dx*2QQ7W5pD3)4lqjWMe?P z+DgCf5@~}I!casnIXjH5#EQY9Eg3e}t){Mvw=d)Ms1ad|WW!RZ=e@5J(G&>|pQQnu z!iJ4nYdyJR>v-cJ(Sg^xRInk783(QMW+O<3A${A|*u{xgRQW3R8Y*Rla|5+#;tB=1 z7ecJ-DS-@fHEbfj^JE*l7hY!0b{Gab?X4u>JGKrWI&WTxY9(^H37jUP2zL zq>k@72tuzR(dpx8wp+~UrsReVMyX>(3&nN*uM#d`1O2}ddMq#Hlw1(W^F_v78pf>W zye^uoYLU{DeysUD%^2o0(%ja36Y~xEGoD*ukyj-W*xbIOJGMSL>++>&ibt3EQG@^4 z;z3r1gP4XR_$?B?nCMKct_VC%v^}KWv}*{ew$r}ID7sU|+xoi{dJ1JljnlP5r*W3) z%-IbibQH-Q4b=d-5toEDiXn7MitI4>gQT2{-RJ}s2zOgDoh{WN>LfBpZc9C9n?;!2 z`ddx1Q4~#4>V)Zy@MNg})l_K-*W+uhCQPGbvJe=e+dBv&B$y?Xv^HUT6)Ebn5K`BD z>h761&dDvJu}KfbtYy1FAca(?Ek{wjXk)Z#2NM8KY2UKwABZUGCX^J_He2fkn~I6$ z$Hy3VqiyQ3D^;mdl^X+AiFQXI9X?#y+FX)h5pJhmx;;zAfyuZR;Bn1Fo$GaPP`-8_ zQPq2cDPWtYcSDmH7}vrDmiE}qw_j9hO*QA2sOv9VbaPInJILBni z=?Gm)t{84xS<#A}VLiloX3C~SA3`gU^0|&6ymp+SX9y;t@gQ6f<|wjec=zW-#eW-y zmqgD*H=40Oj&l1o>WQR0&`BDquw<^Dx?X^8ZvfC*P@53|GUnog^2)csPsN;k8COQq zp~zRrG%;yFRD|bAR;L(0ge+9?su(`M#gJ|#jWmxM8PMdr*wCU_2H(ecA+f@)*(45U zVN~xUCGQv|djhlLxgp|R!CU5oAQtU{tV!Ga_$8l}jJ#OVJZG5$#Eh=7!8X&XqRA4^ z0!kF$%2;(Q$x_IQtZ1_2xrNBgEu3~Bf1(hBqDqX}dSq$ZV_ln8FVCj9D{lLyWbK;} z%nOuL>ptI9-P3FUh6dGHIGo^K|Dqi(mP;;I)@R9c=Uis`0WV$Nl5VEDSmshZkW`Qd z27yRQPKC?@8B;OG={+RnC1O>SV#Q6Ayjc@F1Mp_Gwa(%V zE9pJ70?ep_;AhG2iyp8hpfH5eb_?U7EjEC1=R)3`C(_WTym%#7F$-e5>Uz8 zLU95!);mkKfiBQM$>${Z2QIc-*UIKK_PO2;__zKFG2Cem>G(+C3=)y?v+Su#j=k|W zR`=9fCLnUp$K=$tSH@fC@G<#RqpfV>o|X5UJ)G-|ugL?imJ^+K8kuwdvtt4~t{WoKTXjq4unR^D@y$VtjN&xa>1N;IPtm8=SxeskM+-h&3;t8~wyfc%%ulYo_$l-uG(ozAVaK+O>&> zw2Z^pOAA_7pl-!4PWy;qh0YiGuB-EMZW4OhesC`BO@1ZYV#uje$X$V7+PW;{JZ1Oi zGNn(?pb0VAy6t{aNsgW#P02N1@r;%k0)kmoG;^QdBrZzZ5D02}azsu<2`x%!PBLbI z#WaUmqgD{m**+HABYQW07wIU`I)$`lrp6ip~y^}RZ)&LWs^QrkBkEEcXI2YDVLqv(i@^q>B>GXICIT^>(VvW8C^p?S(fbX+E~M!cpCvusLO_Pim)TflBr&)ml9u+0U(*I14_ z+oGH1L{G7^IgJ|l?)C2Nz}+~a6}rTiZbKqgE38Jdw2)_U0GF0??AC`cXAMq$n7d*X zp!wjI?eJG~dq+dHU+a`~f9L|JHNdY%RgTsrwxq#>(ns34ZPfX@R~Ikeyqd1kU04*$ z(?amv57E%6qir5BYsY-yRPxgJOMaQjj$ir4dxL8@mmZ=p3sRJ;oa4mYB@%u zsg`Cynx0u{zFJ0S6a3;G-?LEI9gUEGW5#e}L4Ck#=~Uq=Sh?bu`_qyZ{W{X1Bzd?2 z-c^XSTWna&9InD2WP^$$U3nIFdv49$Z6;WhY+|d=%z4HDz1y6%!@k?RG4&YBoU!BK z)7iQc%IC;pgJi*k&ZWkz&9Ze<2vA!`=mIOll-g5Ll>DA&Y~eEmE_uFiP_$fZE{#{$ zHsrrie>FH4gM?v!y#4wH3ipD)cxOdNHN>+d2$@VKU{X5yhmx7o?38@R^!p*bp(=S& zk%y(QWm2JU;TeUkiuK;~)8~qUl0WfyY|-KB{KLX}=sn&^zTSdE8*96j3Y`@>jVYC6 z_&gAoTR%S3Lt9|x2!zOgFhpWsV z9YOB`68kUQ^9a>`2cFAiB6%FcgUu5Voh2HE( zR!x?)xTve;OD+L~F}K zInJYdsGr-Re}=*#dbi*3`Ia9mV0ow27Je*0P2IB5ek{NL>LY~W?|9O{j{fiByW%;o zIIToE%5Ttm%9>DWS&0?yFfe2CAa2E5Fy>v(HwiX*p`Zi7m|x~fD+vKNx8tABNX{N~ zo49$N{D(FxLbZ?V&(EXFmp7#fO~qGjD@sJ3vm{hBH*QLPjAwiN$Gl@H#jgI(nr1kB zZ}(y5YY%^X>spVQhBnji*4j&I8Ho0xx$aq2(p;)}Yx_%%wUhIhoK#8S_3!OH2PUg? zS6sn%-r#h+wmC4qChge?2&+2YqNVS*^W(Pv3VtK6xrET1P(6CMnM++379_VTrF8P* z2b!kkb3B9RT9PV*3FD>|1I?`$`=nYIy4KOniz}ca$-n*a2}p9EJ^SMm@8o~nypz$E zq>t)d+*zC%A6wjtAqH#;C1qV>c&*~4dXvNA3owMVIW^+tw8z-kUwLe-NpC~O6ut1= zhala1lgVMG2Y&W%tx^2V;bC5?`{8eUQS|wbLrtBi!*1|5caXO=6!5YP`aMFT~()iJ~#pgvUfW8(=8=c4GI*8E2oFF&3QUZej#_1>N$=^$a6H_5fv6J zb`GwK8qu)eQjW>MJjVNt%%b)WGM`ehf5rHjlrh+A-BtICuB@F5z7yS#ZP|R$2SpUe z;d&p{5JCn;so{T~ZKX5V44_dTHj;^>p)04F*6{k?AKp+GW(wd}c4D4DVO(^meWaUw zJTP1#g=BLm36Y)*6uEIKwOvX(?9qRwwEBIT^mpB=MLHsp zP2Ft9{PLxiFij?!)QgNntTM8>(hh#2rvCxS-*)=eTE)|_JuTyVtXb<`&Vym={+=6~Ti1oX5w&16=^|n7H1E>u4*!1>hD^!M^^$%R~BY2K^ zKpQgD#5Ej;g3-#^ONZtnrZro0l2ZHwphDxAh`Ae$_gJJV%harh+%1Epffa_s?Xg7D ziWXP~o}}U-f1qXRDHa`-=70+^1dG{DbZDM|;5zW>sy}AD(%8=$A==Tk{jQR{aY`yS zqF9E0i(?ndiAXA%nbMZ5dSJ{iO+I<@1Z|H`{`99m5hxIrvV^a?LI(qt!B4VF}=p z=W_?zmP^AK4B!Y~EKLD&wZ<|YK#~V@3fhGY<_(5f!h7sV0$n&G8NXv>E}`_eKbOQ{ zD3?qskzb=%A^JNiKwdq{#Cz=ylS7HJ%^d{JPO5W!LaFx#h_c#W~ zTuV@!?Qv$`#Zr#Jx_zJb(6g;q+fO@+W~QoEY*TZ(VrPom8i35b2 z7GNU=PYHP?MFog{in5*{X@S8Zm}D}`MB;}Nz$+0eT~j1a2|PrWjHaggv=WPof~8YJ zUZ~AD=LxN%iW#b>QPvR`6Fv$e-sa|#V~W~A!5loWyNjQoe}qcH z;LMGr$j#QXacZoZDh;lxX_l=g?1N&@{GP=e7Usf~R$UK)RcE-Ui;`7{+jE5d$OU9% z9Ko%Tq@i#@T_Nhh5Mg%D%C)8)MR$CG!)@ztj;X*IO_F%ZSN7E496@(mgvoMR~h!Lju?x2?#zMNQ5FRps1MUCtef<84=)^I+9E4^*G?5EVLi zsnNN^gl$u&^Kg|q4^^x45Y;*lSg-RS6*~`7vvY^4omYZcjcWza| zbE_Jj+l;{$bv$>irfuYQ7Y1&8{1~cJyR6w6Nkx8K zY4c7YQUrghi+B~^3E@|V>E>jd$Wyg^^W#|VOz$br=qxjOK7=}kA;%VOIt zEkZ*sfY$&%h5N#0&SM#kZCtPY@vSyLw%5h@_EtihYJOs+=rj6^S!yBQYjcIdiJPmn z&fbCMaj)G+P>RVx?J9+cF1szjK#I=48T9lZd#ahNPsJMNn6Ib%F`nIMpcF(onQNeZ zU68V;;gQr$x@Vm?MM_(C-u=&4yin>5(CxlG>I_5&`5Rh{bSc^A+&T_~&O}|A>{$Fg zVz>YSy%NUzwrAt?2-T#W$_k+e?bw$y!f3KoBG?3koX9n$1WqGazEqvHzOwGQGh)<; zvsj&g3GH{WF;-Fs8O2fr88dexfQ=1K?=|x^^gmXP+pJKXajBYn$JTPQ7qRDw>1&fZ zuC|8`^{8{_9b?85>S{p>zhoL*nHsyElcc`nNi4_WSeRE+G%LYQ_E{{bLvu-1)de;e zAO=)#nUq+NnXG7e$5Ja<#m>@|8a!*BD9xb@e(K#c#*(v@qP~i(4cy!8 zV~9~{6&n_v3JL38@p>3u?dVVkceHG2W}`g9&Xj;QS~$#eO5W%LZ{*^kN(Po9t1AnEy7kG4 z7j(=R5E04RkZexV(cRWZV*DYjA02-{jJ3Frv9?F5LwIdzNsP6wZ)pY$f=ER6?ju7@ z;58O-B9d$jECpU4&(vrhHDymM%v?*7kL;i*iQ>_q{1|Z?;p^fK|8ZuoQ44<;Yq$xn zzqq;_X-#mR9Iz)icl>HC3YTb6Z~~af%cfi3juC&NHTvz9r;Vn4sgBAfjwAZma3{19 zI%T0be4FB+@lGL~KMaiKfC?GV4}cwYuQqpoXHGwDRYhw|e5>&i23{t84$e42^ITLI z40DF2_)(5TV#A2tyzJ7+J{u)FZX0XWoDVw%t!Nr%lw`7GD>_=vQrDMiX0Eh&*+epv zij^x##C&}4hnul>bpJFm+OoN}4v?6euIx0lR}O$Q+AKlE&c-S~#6bhF7Bt1|?{V%v z(y(k7{b7f4w4gjPoMU%iG+w(|WO^(y#=0_UmVB0&Gb8qlRjz!N7-L--M{^IUZ5uSB zQr$+83|V4eu0~M|TVh=6$|%wvOH6dW+$PPa{JGH2waQqrJXs}C?m>Nfvw}?>GXKa?AY1`T5YSV`Q+aS%M+2B9H2Qb zM>bLk>5UD*wP*pGw6>n{I|lnVrh{7@ttKP;%bf-0MohM-{}Cu#-OH9F(eiSWF`n@i z*V$l-oiaJn&v1eSrb$tsX=2kATZwXgW(levM;>fVYcw^8HX=z~mMpKb^+%S>?fAuq zkTh3<zP3w2)aDY_$A1x0Pt+S!0e--?IFk+|$yF-_&<)EnKJkp3Cu>5j0^z<+PI{ z1e65D?myDT-A?@rR^=aR-!G#5jd!ULL8Za;GA5&Vzq&$9dmgQd} zRt+KCBdIQhSV2dyGJn9yA%u|-yA0n5S<|dWu z?UY=Qk|n(08YY3__qggb@Nq1^<&|XF9Hk@B!$Rxfyt zN&!$-Qew6yrKo5Xy@EEo=&maBO7)URiYT&GPf2;CfK9JsR);!R5;OHFCde|w_z z-$d_rb;dxku}Y9EK5jsrPII?atyetzoGe9M%IV8-KBY>Kr|=}2l#G5|k&IDUk*80{ zisy9|3sK;xEPwob#5RmqR<;B+a1|8zY_*hVR-vs!P3;5e18wxW-&@k=F-g}M8c zq}fN_7#8J&gnq2WY+c2b>d^PLq51x&{&J&y;9$F*vVyvNU{gJC;Gq~DToB-ZSPD0o!}H5tiVNIZ3;6ZF-8By78U%w6!%m%xMT}1t8&bU z-~;W&siBfYNmg}M@glRM7JBWj4_1C|2z{QnegW zFjCzd;y)5i|0TaqSd1*HXztpHdvEEcjc%CfbasUX@YQ}_LD?PY=iSi>$_>Qe15HPyS`hk}2R|zok!uLwpc=yH2Z)C`B+(tL}Zz+6(i>41uK@8B0T`gH&@dPcqG8}N|vMzEtN39}dDb&6xdCDZQ{Ha-lUXnS_KE~+A zx^H3}xZ_;6AKNKA)*Tz=XdT7pf`+*5oQtFM#bZzyTMQDu*q?OeQpAmtRL|W%}N>Vs@>+bm}t))R3VX;a}OsLvoQ0I|?4_gSRyzXVqib z;$08z1+CZvjX$&X{bu-}cQj2)CZ*w^dX;#)cyV<}{)(h_uTx%%3dPTtVnw+(P@!8x zxAmaulzYI&bmq#^A)3%!ln%sit@Ku{PyT|Xp*KXLUr%Z1@gh?rF*$ZQlVX68J8F_* zrWXz!8_7T<%OwXVpcBiYOZafvax`qj%pQf9Am zI?Mc?*7kz7kB~1jx*!bxI8V7$|B~zL*WgL1bFB+_M*(~bn*Z92Qe+Cd?^R@UqY8gJ z&k&xcrXZ+;AE)mK2CGw>euKeAGe&NfJh{7aoF}4G-{sC%(MFsR&qb{>-L0!Jh6Cpr zb7C36>{^c!{6?h|2@e(mhP41zuI3m^;TYQhus~10y>}q&#CHtksGd{U%)Y*Z`dNo_ z>hj!-&CZ>{brs6??{O8qbrSGQr|dEv<0mq?W_q%X*tw*r1=;BiN0P%yWi|?=?jGFR zqy@*p-|c}te233F5mtV@39)_VmX2c`ztip|;4OtH>dY5}<_z=B{s(;jE^a6%$0l9} zcN25^v%8{@uX?h%%&G!; zhC%N_PX_v>Hv`?|mIuZ&D(`$`F7{xyc{q>UPuI;c-)r{xh@5dZ8{A7XCR2Pf2Rt}k zL;2x>%x^#cYiEV;rz**|wMZfyZ6h|PIJybf?A<2B}B#%$-9aSdTbwsV{#D2_RcW6RI&`9#KRr!3}ixyxgc0OAq= zqO+67fN6Sjk~i~=BPjM{A_udJBgk4qz^I(!Xhs*8TM*6~j>tUNTQWghrs>H1(k64% zFWrnUe=x?FKFW}B+&t53o-w#lLGePzV>XcLl~6qJz}hj1Tc=?g0=`uYiW+@(y05Q0 zPE`A~5wx|Bbl8dd=sVqq?{uHhq`#aOFa78A;>i}arZL6=^8hhT%wj5J0Qa@~@8ar` zBqf8)VVX%S3f;l8(ZlVfs-;CE)6uG8t0GVb_tA;>t9Rn<#UZe)S2Ulf9Sm+f=T!BM zK=%-t61`XPXa&n3q_ovJ|FLE? z#1GfF6XF~wXkS70W<#NkR)vZhlyH~9X1WKtns?PdolZVuWW~!;l-ir)cEd{(B2Sc_ z$Vya=N*$kY%__Y_VEI*o&>RzuLbd`E7z*7tzTz2M(t9rI5-C@L9O<8q)LFR}et2(V znQm(S!)VH$IkT%F!pNKoW0;#jo&ODTcDf8){(&XNds(Qn_FHd?S$hk(PqZb+`)l}V zTe(rA9AE~gNhbLvWJkZ;7_!j*Q_GFk_n+}(efe^H%m6-TP9JIG1=RQC3n~L_@D0uB z0*wqO*H>S>IaMnKtVfqGyB=s)+QJvS-94?bGX$ef-;E~^FbsRh+%)G`s5Dj2$$8ZU ztia#^gN-8%3yC>B?^o&FZXF2+1vIVRj=}ogFoC(~`N2zb*e3MAW}B~RmjeU?4UYZ; zn_*^qo|25`jLb8-fRLsf4RVSxU1Y4RRc-a8P;X0^lnk7Dwy4I?QU)<65oO6_0hmPE6X*8mSd!bxg*#eX*=d^MSp*HmxsX28oj9#idv&i_QkT41BUc>Z+XI0RYrZNv6GLdC z`73rM?4s)QihPqxR;d$pmeKsqf1XDwi0X!@jGg-iIOiu>2tfyZL4F4pVXHu4|x!;)l@;h_cHbW z>-xIM0orQr|JU{P#7QG+CZA^6_H)0>q_tbL`!GeAVpVgWMT$k33jurM{F$);OvzhO zX-pIS3zuLA^@#x1dI0C+&Fkwa`A&)HH?OZzVl?qyOQEOcts3iNRr1Axl|BnY*hdqXqKdDWs0Tu*Ta$BxcY?gE zxtFfoH>5ZWw7OrN&<(jgsKQWXadyC<0{CHth&5D)_=66RPHY)I@2?xa5YI;{x=pW`t>0H$R;|gRpb9DysDv`Kz%0 z_cn2*)Q3XJFYYPNKxx{1HWf*|V!4xKx)AnD8q#XbHeZO0<_pj*i@U{nQm%_ioL8AV z|7~%%m?R>X70oM85$a$T$p}@`7uTC)oaNOI3Ef7resQrOSR+06Yo8L=#_Y176pwPb zdp*kKj|XIk=%ZZzD3?FV<&SJ<_FTOMlJ_2>!Plf{>a1jcDM$~iWC@p;D~E& zL2L>+$v)XrkQcTA7%Rq^oRTjgOBCE{73CRVTQv8HutzXLetyyF`S7}rSno+)-m|L) z6)kocUuBv4oFD-mh~h>+6L8p*S1(@v?Uk>$G*9J{-mx?FKw%p~M47U3>gp-@qPBwK zp**CzL=CWd*X7G!UNyO6>+8VT8#s53^j6@0 zBF8Q6upEC-!Z8?L4KasgY}uvI;pjgYqYlMw4#XAxpeHYq5t#T+$H>%RgFz68VyX?i zkpHB+d42u#vyd~{yG&L*AK3w(@3{ldcIp5gO(;FQ7;h1LA@ZseSqH`tWfA0nx{?(f z?Z>|Etc0v|bIOik@r<-M{ITrZ?6Va)WDvi!tXY%jWC=D$^ckU0)w6^^;qXZeC%R&n z8khjept#1>Gc~t z&S3ac{g^p&+;|9*WVNg|5A}DZoL%p^wBXgUo=p?6a*n8TjT~C7&u5vKov)}OtPZTI zaGrNkJbbIBJIS|&?gv81X?c1{aENko3HuLELbw3c{;(ix6hklL4z8( z?;V54(Vl(Q7B56aGbfteK2HjfO3aEz0mv36UGZFv&RQ4JJ?)uwG*FmSA}*tbJQQ!@ zi#1U#;R7ndhB&3;@?ll5-bI1yZnJ@d;%k>@VPo9HGt7{2URW~3jFcgWQJV$SL690NJP!3#6WK7zzvVgx7&QoB+x*)UEO7MmyWp_=2U3Ncju^< zrrCqys#!UT^u$vTGB8PKJe{U8n(ZQapt^=4RqAeCh#a+l4vsB# zKJq!a?R(2m$_v2b*=!r~^w}Ai)%e=mDXySt!y&%^`*d?`yrlpAtkVT9iKxL*spAKs z0IFiyRc8SYl5Oz<)Ai8u_SJR{138t9%E(pl4sLZ3`1ig(`;PK~)?BxaZ`+uyUtM|Z zUhUJuDHYhv{*0-;T{84gZ`lfe(%mLM?Y#}hu=gkR`o5ESyR^NI_LXs2feLoV);&`= z@b*m-_`((KRzfbQ5@!QP`MZWK8RdZHYeJK%f#GAM&Qn%qP+%U9%f3TaTMfWShO=9+ zQf)exPKgGL$ymiKaSO3&ZyOlQev!M}PB_a&KB-u_;yI)TgM`) zGCE_KBr94a8np~`Dp8*9Q#z~jUByy|LVvegogAFuy><%Zm~_hy?QtsMh+Wcq11%vH zTd}&oI+XU;?S8H`*;G3^zvG7Pd|CG2+p}7_ceIB-tT=8qdjbRVo5810K6nn=Jbr=M z?d~`A%*ybFus+nt|NVcz`IpD*zx|agX<5nhq)g#|lR~7nX6Kr&vVFYc$&)8f{`7}G z;J<&wf1f=29RGd#^pDT}^oM_W`uU&!@W-c5KmYs>Pygk~vnNmg_=kTXPxkTh&{fNd zmjCkP2`z*u@9E%uvZC|3ht3nc?7zQ~>$lb-RMLkjT7_!KN}c-+2BN}h?Gpr35oPap z_fMvOdNzGxD(dy!GfB$yTrV)5FQ$nonNX_D>WA*i`^{uB`Ay@xcur`+*@ued5H@M6 z^*#N~9naI}m_Ot8O*BzOpX}@*_NoV-b z<_C(pn1g)MND=);BxkjG?6ay_S~N+WgfHDSmg=6M*2QIGeCV??h&kYDqijde$skQ( z5iqEhUUB!PHTu_%%ZddZM@&_UR~cDY?wa3%X+23pxONyUz(wBP}Ot zVB@C1<(l1_H+>qKgAL3VdVEcsvzS{6|4O7mdToYGbzxRt=dJo0#b#`U2)JmS$;){+FCLHrS(f&r;=4lJn+z~R2>4{Aj zQ*!$smdc;c;q#wwPkVN($BcT%Sn;wHMZwY=}xm{3A!tM=5C1J^&sstCHAQY5gr6T~`msT7GFk(X?NZ^BT zRAg_jOrB_iAdQIKow(RuLf#nUIN#xbkIW-{6(W6ur|B!L0)-AM(?z2)Nm9_VvWI$e zu5KHqa<$QJ3 z3(<(TLUrTWZ4c**8@W#NqcL!=wZu+M*f});#yTF+C;c@SB5b-DzwTJV3D8a)@lcCR zWs68;J6IaqR!MBpiB*pUEOr~e*tE36#M(zk+s7xQ#iXN!lF<%Y*TWZMLpFG}i~ZHj z>|Ymb3R^W?Igv|}P{}4dmkeTO?oqycfAodouJ4Zw2px40Ca|Wem+t%A#LGWdv=}W} z`Q8hmxR0_2_C$z0_nH%=d*(s!;pA%u_ER|X+9&n?&b@+4`5;Q~?r5{iTl2`RQ!Zum z2y7C>0*4>7ZRk7ee0=TkuzNi0b{oIP!*1|h2RiI($;!~G?eH7($lmykA^6B|^!3PZ z9{EjQkAC(1CM-CDfO;eDSnkh)?+TFJ=e*2mSb%J2(IZt-DYC$v8bQ){b8jqz7$llA zmKy64cWN9o7C`L>+;BJzlB9V`Z1#>WdgWN3zc!&tDE=2EOK8Q?=W2Hi((D?acrGhO z)19Q*0Ovgj*B#AqqqxPmGxAg7vU}&SQR6kbZuXwI>fY*Phl^JYbI}M6I-YwnK)nB( z4V6L!i#mZz zCYGVjC9BRfy1(i%R8*FF7E^WGAkC3{La~QT&{UqQkx~DfWPHw&b&|33->N^0C6#Q5 zk4H?I$TIQpj-|X*tS55yb)IShs#N_mRrN~7I z(KB&9b5`T6^Xx5wSV9erWFk;JunyY){Gjm`)+7;G##7d0P2<{v#6@+hKV2`_#T&k2 zvZAXZb^<$3-I}23W(EU}HG&`-&Y)D58upugD1>b4cJ9nvAOiM&_|$3hl}mj$p~fxl z$WXX4-MJj@qzw#nY>N$QuS2w!n&Ki8Gn(m_4AM2-8J59lRk1REPX6oU$?mx<37j{rup3PMyz7=kv<>EZy(r>0eI&)jJv9d~`Kv?`*(P?6TWR6dtu4kb#BE42V{>G&3nsl*h5+bSdpYSQ>zXmT26Vd4>53<$WDg~WID=sQ zY7AtPta2h_h6hVRB)0UcZj4E7^j5z}OC+d!!1y z-sj}@o1$Q4LM6kn%iGuD!TznYZ?#kz!#VNNF~NVFKADo+m!&9P{aDjXzu+sz?BDGd z1gLJt>OEC2XgiDE;fVUZMR7H|VkT!o-bLMY$5=P%x_ir4;ail57r5nR_?9J=%a!mO zZzkk#R4(5H?}DAPnNZ%n4Wx}X!P|gZL??kw|m{{Z8Yp(#VZDJ1g|je2wrd!j2Ogs4ASBZ zQkv==T~MA&BW6NA(TA#}o`6Mj*&>{968TF!L@x}@k5?=o1MzK61gLwC?P~XnMzL_x zOnHMsLAbG&ZHyguJXrkjeXa(ceIN-fs=8#DSBb%)HZ4D}c4Dgrvx0(wbre3jex@lp z4>o}rs~#AG=+uN(EFT2ZYOZV8fUmtu;kf=d1V;kA>Kr?*%xY7uE;R~j5&OVp6-M9I zDz2i*OH+XwqGFtJA+%>kBD-$eq|E9IkZjhbfAoWp!DJ(H*2MZ_ zmLpoR9bpZ*>Oful_JuogMb}XO(s(w=#r2EJOOi3AH_9`e6rr?>u&eHYWf`FoacN{C z%J80S^waO@jXMHP#KJlDSu#?Y%b1OAYrf znXEHV=kzw_5_GJ}V@Jh{=Y+ca*JKd@htE^BQW3TAR`b%^5E{Nc7|&dlBnQR5$5yId}An9r1Mb6ZFS8IUtg%Nl_IVt+C-EMw%QwMHk>A+vqT2T zx?yVL!qJ+by4@nNA=mkGOhlS_#2+Cd{Cu{zERi!W8aMY?lFOHS2@{E|ophM$Ns;xF z#uP`1ERV8#6{S&SHeySfe9A6Wa~+_pNo--0kz*cT+~hD-mA0#^G^WZ)Y@L&sN+%;~ zooruWDXzxS;i@csJXW%wT1#73TY6+q|6J^8o9avB>o5Jc=1ozrKmjaNtY$zs+$Bnn zY)0yB$b`rs9>a&dw?2js?>2sq;lqRPI#9ZT$?d+PCH33Z8rY_*$bB9;?HfUMZi_aI z2oS1NsNYz{O821Q!USF_jnP5j8XN|HG@B_(k3K<-l69`>_F`kX{q)R0 zAmlw|gMxt6GR%i7VeGXPy<<@PDq$(sr@Ut+xkcUjN|e>GSvB0R zQhZo@I8S#?!Rcw)? zl07_HF#(&NaVSSd2Oiy)QE`D7|L2)x6Bp=4*;;f*{vs1ap}~)y5|%z@qYNB zS`|Mu@q0h~L(G1Fw9UcyK$Cyz;K0X_%g2z*$B@g%kjtT49z!mB={<&AZrt}6a`_l? z`51Dkug8$f$B@g;^%!#b7;?FJJ%(I9hFp4X8J=&bP!m_26s?l@fE&AFY);VeTcsoW zM-Fb0w=f)dz!!i4ZMb2bXG}`67PU48Fk>6RcmME1z!(DNl*X15k=U!1<3X}`hGX9b zC(fteO~7{G{c(t|gbUk(o#TeDn5ZM|uKH#|=M^hAv#1l1ORjjAlkM*_%gq>>12)6n za;PiaK)n_?Y#yg- z3o}NzpTzK^KGS~bmb$Q$hIW4a6%;lZoHSj-W5$+T*F?EyUiIV-)Ya zOC8BmdyL{;eSBcFDe4oHNs<(TT;I|B2e8XU3wuv(w%IFjoabHkWLWa+j9HHcVXiCxKw4g zqezKuA3gb$z{y=z&ih*7j4&c&6EFH+->&N(5ois-B$5tDyKO(TU|r%hfquSk@{A@_ zRn^0L;X0E6Wo691mXW9E9~gSsllfc|K{`iU)y?}^okVk-$%uB}pZ~41-FN+b`fq{v z{Q6@ra-QB)AvU4T%!jwF=bQPRWl4J1TR{oTJfLGWkfBNh`>;!Bx#v8x6lZAXA38NKF7jZpDv;pd+{iD{lJA zX9P=S|13y)+-wKZ0ks|2qFHbwozhQsIv{F7^dFssD6YBs;dSF6kRX<-!aZ=mEo8)& zp=Q&pGST$9*})d#*&d@mquZA`VYajT4SA!^rf1Vf-IRIz;YZW6>DjIRLp3K@MC*4B z@aMN(qF7(()daPmc0SWJ|Ck41?t!nC^WGm}YzaacmC8y&ax2N(=nq}-*W2N=<-;|_ zLtncd=FlPf_|+o_DZVH}f_p@m#=P3Em}pp=B({qTWO^46U%Ovr12c^%kvjc z`_cuxk;#pzo_nTard@6C5(k^vNqfmoOog2j2+GC`ahKK#d`S>UHiTpHiP-vOn_|-Q z%)a}=X@!K=l zeFCw`M0dMoUWRgat25D%E;W~Sd!StFb!ovnLm%U80+FSDYliuD1HRc-{f+w0)j#Fm z^2RjG6wu9*x(QP00@c%+D5C^cWD)QvtcX|;lxsTYTh+TPv2vdtdXsY7cUE}4^8<$9 z%fA^?Z6vs3+q}qp+R*bA&Nb%$A5ESCiT(%Wm2)&Tb`iTOD>7Ef|4xNU=Qxd=HR4pLwX?UhcDTIVLe-yvgA{c zHFE%eJD6@U*dEvcwi}^V_xy2)jmT!XL}{3{XX_efhgw-vBD-WhV>WtlwX*vXPfE`F zQE*4MAxt2``lH^rz|xl1BA%t<0xyXD&7a0`-;pe_Q1UbpDg56g7kozKbc0vf-o9~m zc6RpV7hjnF{?+{V?9Fm*$kIp{(6FS@5 z3_PQd3X4CTo#C8wvBtg6NrmTg51cbI%Kv^KuK4Uw%40J0)+x($)3MBoXQ9$~3p8Y%xu^AYAIN{941jk0z7JN0r;;F~T_| zZxms`gr;|Y1Aor?(MNZbrH|2*LMpy`MNC5MQ%y>NV}Il-`DlUzweyXXZRn5fVwn*! zSr9hWM1EG#EF}VfyHBsrrjI_eO1WFbv{|mDeAcZ0BkR*bc14}b{YVo&JrT&{dyKjt zGLYPaweZJ9DSk{)#|Fb+%(h(V&%@7Jz5TBG-7{+4I(b$IoVke_0=wiwy{JB_KQc3a zp$sx~?c*QWT7r+!3)jC~b0@nO@b<*+!`j$BAlI%;1K~Mg=U10Me0JUVb&E~Hd`s?F zZIUgfWK|)l4>l<|p_b6uLdz_5?X+7Ttzp?m{mfcV)I8O&M%u-a)UM6AFO2MNbUZ+~ z7L(<3P`iQ4O(FS+2=@Yyn*%K_ugCBt2X*tmw z8+QbCYT15xOEz}otLMCof7u4+YK%{b)S@4{G#ArKjrxyOrLC%ac58bZ_T{F0-*07V zv_c6;XiFLzFam!L-~EH9HL0R`O96zjf|T`$5-iDtGN}b(Mc1V7GW^WY&j(*)Nt2%s z^n02e;hL+naxB(r&wWj>Afm7IT!e?IbNk9BR3W>_*X}`6#u3a^CI991&9RkO0dmJ& zTf{E1>Y|aODOGO+~y{U9lOBqxD=@sp9=Nr;zF_=!Y6f#@a< zd-DmqDq?}_erX>=xGYs`&DmJkyBJ-B6P^|jGow|$s)|*0Ne`Q4)5=+vmCLs8?(b2u z%!6nD;gu%@m!X6~PR$9^i8}j<^$IZ$iVDopHpw z2dKl;W5Oi5Hmlkwo2Sqj`OvXI>oWr9!B(!tN}xdR%1 zO-%i+dy&E0wY>m5uWv|~WyEgLjvLYi@xiv}2b>lA-Xmj2D>tc2rXe57CK>Ye%GW=q zu|d!4&zm9+vepNOmsu(^GB@JMg2--fL0lTau-+`@h&tK(4qaJVy;nNH{(k*fLVCBq z2TNr4l?Um&vj2`NJN8Ab$JE(Yjr-Ztug!-ifJEa7B zq=dVpg!`g|JEDYpp@h4jg!`X_JD-Gmo`k!dg!`O?JDh}jn*>cmfqo`Cb~5phdsOBT zMD{R==u#5XpCnwo*xQrjA-a*o_92OAvlr|=vSZf~dknox7}#%QtKfYv;6a7(!-da7 z-9>ipD-zW?Bs{-}ZXvt(32~i3dN+G`hmdNIdV$9G28rnYF{Eo?e8&%i?H>E}fLL#Ur{@M1SH=M9+=~>DIABpN?85j$W|Q9XdLDaQKAbd-UcA9Mg`~Ab?wkOk;wrh+iFu(^s$&nfwrXP8%`o5(4<)*My0ectqVnoaHCMxB zb>d2J1q*Q9r-Eu^re(NP*8Avoa^nHYWJT^556cN2i zRp2NkEOp74CJIk$fxbAY-fe^o+L+|9!&~!cvl^bIXXa0)6q~;+v6-&TQtX+s*Mt^s z9_Y0x!S}xxF>zssx8~tj&$iC$`&ZBO{?CcBMkaW&&*rZwYxhdM6%B){^#znUg}e7p zh8T4G+3iq|;i40hOt@DloV1aV=_G9PYM1#Z5KL-A%=A%|x5iX0GP^QGHXI5iWg4sv zVx#j8m>HQX)KiKnlwi}9RD|j-^3y)cRXchFtEJwHglE8&U2K@)&en4bZmnAb@^+`F z(OmF1o48x>(()a21@F9iw$lfFEqNQC3-;r!n(?}(yFJ%#|E+kt8}Z(!3vZ(bZ|5`p z0RW!%yT0bTfrBcdkm7l11lQNkN@Vsz?w5`g!w6A)f!*us$&#aW)z*zEfaI^MyY<3Y zR(C9`JC@ZQ%j&*oWp({6*PGPEwleQJ{iFE{f9TpRTdCQTr| zB;QT-oELM}=WvSwUtZQOm`DVPm$r+jUUQ}&mjS~{y3&a&Es1LkwBP?+@) zaI-^EBy7Y2TfkcoO#u>D3u+bPY6-Waf!L&4MYO0@KeGkJtE5W!SKH$21sGAGcR-t< zaFyODuRwL1Z1O;?_1KegP?hvHq{*|6$}3nVC@2Ip$r5B!DTNEVpjvIVbi{|vX$qG^ zUws#?16rn4OVuj>wF*7?)dKw0zSiF^#wx5JAt#tY88F)fD1mx~^fyu|ZmEeRsn6X5 z+4ACl@!8>_P5FtO{=fL_S12?Pp7%S*is(1V2XTML925T)Y2aaOu89|9XCBJ$m~-Mr zY}+}14oYP}SQ{31wMLE`)M)i;`#I#^W9Y-ifrUo;R z2~&d^{bVS2rg}1zLsLB&%B88E4CT~RPlj@9swYD^Hf}_VY=N8`H+Iau3DPBNa37L` z6NDMg(QzXtSI3QT&W;=2+#NR_B8SI~*j%3G(jt&0BDbe=eepRy!L?PXSVMAsVmSIe z0_@$0-pF7MkQ+Pa0tHtblPffQT@krMol6^(Llj3<(F>Ii?Y$i8$$!ZEN$!0z6lHC-~R!`!y zoWejmvY_5AF^-B-4WEC1%R)Cg3UO~WE!?veHLiDI=LCoJFbo_-qr>6gUWRT&QNA5} z8-__my%>5-kHa9uRu9C8UWabPnpIF-M}M?i55x$l$n{AK^g?uF06mWDjo7)o*dB>d z3k1Kwpk9elU~S{SAw3hz84cOQa1TW{2KQ2oT4zVkLvK&TD4^SY7rng|qhR*+U+nC$ z7zHu952NX|7zH!fkI{`tDH09Er+7grCcDH8e~yWJCeReHIlTb0N73sD_H}e)NYHt6 z3K2WMW5371=Uq3*!F?Y?%c^&UjP3szfZ5RRCuVsv;tO!umtY(1{+XE}$Y9I`J@$G4oOD7uz}AD+^GQp_C+w zifo$;WTTEL&ayK0%XueV44CiRq14(BDHcHX1_O*v?9qumILdplRnHRVMo^q0N;)yKb5(T^$3E^Qu;9wYKhqOK;)`YwQ#K{@k>wwVlY-#$bq6Q1AUNs5e&D94l*%l{H5~Fjm&wwX!A=F%>nR zK#`WZRAB@|h_y0;Ny@>~3lC0O{i8<#WE4O;#|KY$Mge3LKt=&%Bm|=XvZDY(nIuUe z$n_n~e*oJ?ggEk)O7OwiiXL^b1n!nc>Im)S9oa_v6n&HMY+TG_^-#9xvJNxpBFU)`c zYW{on>F4m@PamC~efq_tKRx>V%P;===+Wn&fAQ!~XP-WL^yR02LT7tgqjwZiVezN4 zGn{iS*0}dMsqlR6fpca?`QHy@d|XmsfnQHeyR||RVSU=I&TA7F!r%G3_1W~xPp4E(n+UE5CMP-ba(kPksZ@@0rk3P~g z`!RX~m9<_G>m7gi>-m%ZQjP2%BmeH7jVA5&^*7 zr`KoGN1xd`qHaCYX1r_ptl5c=EVD#gYwO&Ck2HzUQv!G7W7PeStrKXcjulyOQOX+= z)UoOC7qdZ?SN?qA@ssijg&zJ>EZD&_Tkp2+ynjl z9RImE_O1(ro;h2lbeeoJqo%62fXZUF(pr=4b_1F2=t3Ow1 zt2gQ0asTuZCzz!ZLkgRF*l&EPq=<9Tm8RB8=4m|bGO;94fYB8!gvFvT*{1W(^k%(} zhn3WG=uAvR)Z*ndnP91H-9)B~Da5I*aZe!XlBOwPXrAGPbivJnm>TZ&W(($QQ?RC` z&5)=LsK!nU*&vUmDyg^hXC4TaSP!F4#Wy+&MV`pra;jN^0y40esPgFbU+ zu)$Y^9S?X~)Dp_AwEP|>|Jj)Yu2YtvfNQR|Wx`q3Z^f_fAD}CmOsqj@s$@EwGCs+8 zayO?LIo@q$J|o`*CfG&Lc56ppkxbM4q&Ho-CE<0yRubf@L4DN^v}4lS=KUH8uh9_K zu%}KcwIl++?tt5Tu;!ib?wASoLKwjN#(rinjcI zbzhHu$VlB+2T|h-^dib;LwT;t1@jvi<<;Pe7I&`==BsK^%rr$jbw5mU?eak{3~WG%>=lGpw;UO_UVxf{~Z1#dQ+iBZ2Nvx=}PQ-GbNUh>x#X@JkI5ERS^rbvO zvqEW(1Dep1YgU5ILBJQ+l$)Z1oDqNJvy~<_mH^eJWYo3_AeE+TUb*KUkT%;>e_9hF z_+cRkux=!amHymoLKjOVPY|7wsnJz&vJ7menaYX~1yWl{=xZ2Pui13uj6@!=)|_)v zQh_!ubsm^6`;4e$dVGR3`z(|`6}{rwMk`PPTO*d>#lmJuXLhoI+BELqO$o{rl)wYC zi3d(7)E~0B#jsm8o7f9ib9G^a9Dvy_(e=Mda7XgEJWdbDC0#7tIlHc{P0YDR&RA1H zVpatzA=fcM5wVKc0b~%#_5g$R(N&(&ger@p=nNeJl2Uaj;gU^we(Vp4s-w zTGDdghL3omx^`kQbBHZwQ}xCiSa20EtgN@H7j=TLlsuteA&S5hGn(jUwtSy;QHK@A z?Odu$eQYkdPHG}xf+D6N97>n(9r4zH@{Qi}Y1hMzf{1G88?K5Q39?Z;_a3!z%@~3S zW<=J>HDfyq_MT;O%@|fJ*NlPHInltvxn{JgeB8fkuAO^a?c4znt17#;yz5D`d@lM{ zokS&cSL0b!(OtWs;fE1lfY*#RtLrJgi!S~3+gkIEE5SPil^a!Y`wQ_709h~D8z{#+ z0BW;vZ&#h*Z&hrKU^AZe)(2kVH3)uCRdoPFD-51ke}S{fgJC)sM1jvFp41}iHC_-g z4OJl?0J2rDIKCFKol!hXGh>I#Hl#YS8L>5r&4{T~?4BHS1EQ-I*B_5tB?v;_fJNth zkb$giX_SFR8EBM&Mj5Dcj55$D1C25e8fBpH<1woo0C<VfIJ zTq)wr`W7@sO(SRRfo#`fXPrO@g5sLMyP5Nqo+)L@k!3w?;&#^q+5|SG1+jfcOhoRn zS-adwMOLN`q=5hEO;BRv_u-K>VXDAcwn1b~ZRa*UAqX6%%J@F-TXmp)TZc4ndu5Sx z+N+}}4KZ}twOkJm0w%qD^$;!Ez$L$ue@ol_l9qj1WaAVZz{bSVWJ!91AN1eVk?@^~8b^1}K z?->8?I{oO^7-gVxj53gCj51KuS1mOikZo4aj8)F_H3c>gQlI%wzGyu*wd zjjk4{Ap5;S57#)(xT;Bd)m)R8a~O@-cS?wvNO&&5lj7n2H zMPC#U%pr-26kYeYZ1dVNE~Oo8lvffl`Cw&8US*Lr3>w3?LtE>UZ{N}dR11? zXd1r;fkX{c*-i#9w5uw(iy;iPx1l_Cv1Jt-xDj<5TEAs%Akc;#ZQD9F5HIy;nA6_H zAq_QwYTJ!^wdUd6afdVA zmfXyQjnuKixu!b*M++Blf&M3+yJIS()*_7ExDLY^dupGH2CHX8CDQn`#u;JC#y(n& zSkJaCf5uab*Gn#mZCIkUW-8Ia9oZb5wCPeb#DhzAU|fn@G^mu}piTDy{H?bT-}dMd zfrp8<$m6H-AT}Z9a^c4*Fo;|>s381WB4#O5bMrR_zv26 zE3Qve(v%2K*PZsT1-#7Z6}FmL#`#^5HyZ0}?12Jhu{XP1rqsx5(RyO)3)}wv`jfzO z;)E&CJ#U}YmF-)ECAtNfnxP{AetcVPSDcSvX}gE(!y%fxTMLrx*zGuoJttB&F>xEI zK}QO_dOMFWQ}npBh)faS7AQ_BXNStn9?>|5R(m4uR7y9Qu|^wRHzjjiWN{m6&REk( zC)`-Gy48K=T+B=fT*b|lV~ra#u9oeqrAb=@C#L8;lRS>*wqPY?ZE1a1p_yg_blg0p zJ_>2d@X}I%&hjO$*+MPlj5~b7@C=N)&G}xknklEObhyAOP}IiGGhS#L$kvC6DGfTE zQK<$JYAWwbtJ@Gpa{b`snsL>n*ij(uuEtaBD5#!887cN&DE8I`yC2cs0yom_?(sg; zZRR=#?bC5$tfD5m)WMHCsxtl%(xPex)iO^D5CVQ5NB^vMvtYGW4b`M9EsM%GtVV^0 z)>Ccjx>t3eYaf7TiB;BQmbMeeDyUiPK(5i6R+Yjbsl+WgC}F~@1hO|3ehF-L){Yr~D$<5FL)>D_UM`bjG!+NNaK=B+bQWpSkF$Qe_8fy&je9SIu%n4*&s zN);xY^E3|08dbs?!M|k|NtqZ?VMiR{eZ3^2v0f@&pB;oH5(R_)XPnu*iAbQDk{c_p zy;nxxgSCUwSDjb<7C0gRZ5nNPAz`|G)~_MF+Q!)$Gg|tJ&GntY%j^*aBtkJP@mb zcd=#d(2w11Xx&i29>5b@WQyilm5ODiml+Ge+~)Dc-h4enY&@s+>tmbyDk3CMaGUpr zv&@vP^FVY)1(QVB*&EG`VHs0T%Let8hhCy1&1kcY^<6va=OsPc+}oJ%*2GDorb%(h z!jI&!U+AzRah&+FEt+^?cZn@0k>Y|Ea4AYehzWnM@x1@^LX8j*V3SG_LHSb&w*&*v zv$3Z1L3vvbk+bzM`C7Z=YQ57ki2HJh4RD9W+X8kk5!CPA8fygY=rinfP({@Ot;W5a zQ}4c1kjEkQ9@h*4Oxn%0^sWm4?a7_Ae^*k38_9lc2N!9e6wu&>_q&fuhtL3+9@o)W z87Jn+jJ;EkX3Y~m*q*j++qP}nw%yalv~AnAZQGo-ZEb(^`^Ro<#9r*h8Xsv?4_{!vA9nPgu^sf(4g|EvFis*qZx1ZeQO5oU^o#}t zxJGC@L2|tU97_xb6GBIp<~#ZT{p*Ypgs?sb3jl{U46m^37<1bJNW`NO{sdCrjb~o^cVx^lyogS@zsBAAnqY%#Plk8+$LP z8b*o0>wE6_oO;rwII`D^Jhd&JAJ8XZ`+yrK$xNkW(Z9@>Xw2m0u&<}vZ8?Oz{<{0I zE*OD54&L(9bR0(VI1FdA8_8w<)NuM633}y;#)8_41+^LVBaW!=(t7^b{to{LH!teD zwpKsdFp%4)(M8z(7ppUR2gh&wE$n{|uxEZ8b67_v%ksY-HqjIT?9n+B z`gZ7_x8vVS^Y!056ca-mSoI_PbbtT%bRLJZAID5jgBV_amG=kE!uDT@rpt&8Wd@O^ zkDTkevWJT}xqI3>t9fi)mk{2g;(Ssz=P9&HNi-3fE@W-z`vv0REfx>1{gfyKi8*`P zg-sQ6nNsl=srp#R718U&#toMwE>KoR1IX2Pad=5I2v87W5LAYF339M$fg?&Na`PKoBHcCwFMfn=s2P$z*-V(94$k@%IfrF}Avo%(0ao~@5AJ%B>jtY{Bxn>nL%>=S|aZ&2d#g?m+{c>Ocb zJb($FaGshaMThgd1oF-zy+TX_@uY7`Eo3Ll7ZXw2$s0&A8s|N)A*Khep0lq}wU?v_ z#3a%R7f9P!So)*YEb{!$?Cl{5Ym9{ZGv~S16yU`I$a-cGbS~$8F zf1x#k4^?pM+jc7`1ZN`juTFke?g1);lgNd5qb3rs*fjy1?EE5FkfHGv@@$ zLFn{_PLni#xWBOcDCt2chct~CPvud5{q=JbP*T&@T42pE$8BRsUdSCB$oGlo9)KN&H}+LvZ|_Xdohxx^vx1O6c^%{^7yQ1klZz2Bz5`|Dz>iC{r9 z(QoXGrjfj+2#y3$Lsel_UnC+lX_9>=bmR!@sL01i5czvk1f_h-SSEvTEZhgV6~2!m zmDa%bQHD#PqTfWXYmPds+B-gl!cZe)(B|7aq8G!R*cRd!Z3l$nA?5>AFZ$4g#~<$? z1S!TmX++Ypd@G>JgX6#&7P*(#T;!=(>bp!Rnj1iQJe}9Hypmy_Pjw-nr`vl0ww2KJ zG=R6n52}zNpG(|jY6Dp`5h@UUJ?W;t^A2vKpe8D9!m%qf2xT>wPM?ULz=pAUrND=fSR8GS0ufQO?GNL_|2R z@~7hBAr*z$=oD8oe1>@vBm18G$qYmdLO19_i5HQ?tLXu6!nU>F=o}cw9DRa>zO@gs1=N8jQD{0Wb`B%+i;Jt?`Q4oz}q-y3n!0jfVp1N{6%X#g#XFcVd%O2Mr z)R%C|8Qu3#)xer#vCDwQ%?A-@0GRD2U}D~M>FVs=mH+mbj=II3Vl7s#%QF=}w+=rX;ZSN~SOmxqT@hAiYE6rk zW2xr%_`C8K1g5~6cjOh|>u#`1DIrNd9L3hgZd5N|D4>)dMECcBj`E;R$*MA>qygR> z0G)~Y?ppe+7wJPT4oD7otic>O%*H7@0>n_L1d1h=1cf1oY;t%~vV7xUP`*$}KY`L* z9$p8S54wv%XyUqROS|~!yAbhJjaJ^)0~K?8makWRLs6DUt5Wr;4G^5{)42&Q=(s}e zLlhsn3EqDaHt2ZVMI_MSyPT6Dh$P`+G@*%Nm!yhf97ku?W@;Pgu%a}V5KC{{k-pX<;6b3FOgBHrDUS?Ij;gH5^-pd@w z|7So^tar)BeUs$Nu(2OQhMdlL?3qF6t%nxvPfL^uDrfyJS49{4^%rUauP`G5JDZ72 zsIro?3^IG3-D8OKJHlb}J<*B${xE?j425@_pARajqy|Oirp%)Z;{4%<0x3%0n?QGQ z)@S%S1lF)#jjt7{Iv1MIwe7p7)_a@1q&nRDV*=+4Nye?SlZ-FX%t^TGC~=h4qsym; zlt0OMSnBHi9&{Qnc07PS@u^*w!780bbKON|J!f(yOiTR9!s)S|^o(>qrN`D`;!K^Q zaP7#+Nze2)?x!>BYmDC4$HGI(e$8%%9=4zsC-ep*tI7SkGLy?3uixzCpGP8Nl)rY^q# zZ__s%djo+IVUiM*C~g{q@dei3Eln=*>wwLK^;*6cGLlb@{a@1KcS;1E9PWy|r;PX% z3nefDGA0_Kn7vo@87Oi#t>PL3`U0t6XcAU*8}r2J@-yX-Q8#af49e}Pt3qtx5--WJwD0RQul(XR#mC^{6Re;zanj^es?eN zVK(M~$2(*q&JC^pn3fQkXvyM>iTnPl&YT>kgA44@Tj`Z(mdY!~S_iu48T08SK+Fj2 zd0d{6RnpV!@mpt1bue&LV4W+(cS<>~-T(;)X)j?yYy75=E+|XGe}OdsP1}UeDoG;J zB`2Rta~{Hb(%n-HW7H}HvE){FuC@m`DmZ?|%t$B7KT*YeaK;azJI)H*C6K|45UGjY z8c#rt=uZT!dyb+fVa12Utk)d4LD7vBl7s0Yo5?v-`r`$+(uZ7Vtj*C!nw5`No2x%X zai|@}%kj4Nu~b7houl`l#xeUG*g3}V6EKMP89&7dfbS>{Xh1ocNO77yo?7wjQ$=A1 zjpNozcqpuPBP%S|NJkrwxAd)3lzB_dQ#nQEISkglB+KdjVVFk@#*6%+%3)u&B6p_K zuF^wD+uf|{YPJ!#P3^bC6&hf0zLB_;`}MVU6mU+F7GbWMq96h?Ju84Sj;wQHm(b{H z-^IT-%~($gG_Ja4#3c|wT`SH+(>>r)llZVJ5LaJ`x^Jl&(adsrG_MXE)_*6PjY-Qc zQP-cauEWEEcuc%kfns_Jva!KHF?aD`q_t-dFi}FDu^XlxPkqkh1yUYANGjqvPPy7P zsPh(a7mXFF^bkO`u|9*!0d@v#b+)qKF$Uc3?r?vWqee%kw+T`vKz9O=6q*P2nYt)N z(!ZFCW{&qyb)SjKUCPgK6K;r%G9UM zeNMnL(3Bvc1Uk>_HxUzr9}Gk`4mAk?gXTviLvk&fStz{a zniN2(5pVN2@a#|^upA>Tru%I-!f%$rXHaitDK=>|R=jchEyzXURn^~AT+}B!8Jvx} z=(h|qMF*?CFE7+j(RFXW> zK`5T7YaR zw}nYzHnz($OJcGf8Nq7Qgnu%AOFxRRTYQ1jW+6Fiea$ycznb6Eop_N_f)Rn`F2DN?xYT6V8*qr@AZ> z1C^I?TqG5sQTeBC>+Sf)Pr}!?ky$+j^NH>!EaK4^RyKLN2&rgU$*#oW>Bb~0fRM^s zd!ZZ^eehB3UpoP}QV&=^?4wNf|A<589Tr+EEd_3H*-*+2;$^AeO%2daVTiI)JK!OM z>}$nTTIxyAiYgdeRmnSZ_O*=6q5?U~m{@C}O|sGv(E>8EL>mw$Qq+T=MNGMYJye{) zub0xujY1VAy>XyXk#qC-amTZ9q~yM>bFKV^iFvDlTo%KazS9%q`#!upc&R8L!WzB1 zUj%8v_I3GK{=7-q-!CZ0>Gt}%oLU;9dw98&t5n}g%lY|x5~lSD19ANQvE(W*y<#pe zI=&bb5w`X=0&julvNZ*E>8Pj6u`Q1<5}&~XP!4AxVk>wJN5;74{XV~HQjqSTUBJqm z%QeuUeE*za9_%XDELd=hZ~D;}^(!j(Wwa?OIq=X#i~%kp`BGx?6;5 z?`i&0_1Om`g|NbYL4{^=;FCw4T3PcSZv80o6we%0(Wa`aqoBHEw)_PUt@Q?C+2)s=- zSs06;mOy<|eaELEVDk@#&QAznh$;O6QzX5_1;Q*5O(Vq7-6mX@5ufdN&8jnw>;zN;$svID zn69;*QBjZ2qx#nhIzZ_3+z&^%I<_Awb~@`FT+?L<@VOt9^FDoC_V#Ie^pSyoUU}}L z^B%$2SvFo~UN0?wOam!Fs<50J^G^z`!76E{=Od)%M4fVHNdBR9BrM6VQZNQ=nqx&B zE0GB?!;DqeLtTMbw`3Dy5$7htGS$j@n0_Y_kbVW1hhBsXa6|cCS8P4QWK-c>j^Z?x z9nA1NLPHUZt7+!SkXtT))UwA};ys|dBIG}*(C;N@!vGbbxs;`r9vU)M$+rix`2iH^ zYslU{e-OLJnR@m3?Y>1X0^Sb!$VjrKA5soaN0xGS(RjPDsorAQh#~6!7Wcn~5I3n} z$41z2LQuj!6Buja7>d9a8a>F~g?YfyLrpJ!q$+@^%@vyqYiwB?je%LBgES*zNtXG! zR%0>8Tw3U6AUSg#YUdRUEDY?^0Lvvp;H8a&@uzFIYqSBzS392Yi^dN;fGD7BTo}&0bl|ARv2Qfju&FyUrAPFlwM(R4bmg;kroD3uTQ1LbIff^Do z3zo;Q&X^0TGy`&mxaUbr;;OR3{nF{o8y6iE7C3Zr21_+I$LvQmah8!y0Wq_*$lQx; z7_$?;S?!`p_2wEzKv;{!fN2RCahr%i4Q6S2XH^EOb?NX^`$WL4VVK}0BDwcKHNXbUPnqrzhE z>Q&KZuUx}B?Vh&DSXocKkVyQNk~uRC)B-v5p$q1IIA7pPV$sBx!K#5XYYx{=dnM7l zbT>wZem|0wy`U!!8HK})wGucLia8EMO`P9wLpQ6^#KT-zKC`4+`PU)>y;bM{J#%Ju zeyefL+l`%*cFR+i8P+wiutACD{oG3$+#NOSo(6n55=!ivM)EAGc>Q7iuIF0^n#~O3 z9l7I`+VW;7>>?&^F$=qR1<|wY>Nda1Y_t55=j9%uq=jQ;Dcyu_&lGli;fV&g2i@7G>UX!PaF_oM8AsZ{0>;%`8 zG*#5H*YL4=J|iH=TFNE`lOWLH`9?$7O4KBdUJo2P9OK04`D`#KB2Tju?>^_HFnGYe%~`L8j!L zRrV%~vP*fklallO!8rdc+3nv$85A&_O`=MZX|rkG2RJL8h4d4gF_pAV_JOf2D{94B zO#=}#=7PLLA!A8Ja3F^D$dzzltbX0~$0&)RX#&BOziz6$8Aj>;9*z3N*6<(^#2yx; zl~@J7Zr;~7MNS%z?|~G;7`B&7@%PI&P9Ga3E@IJGcHM0G@+A z0Uc}XQg0}y?)DR^9J9I`EVqE;pg@~u#zWgl2kmAu)*6F*eDu$=lek=V?af$pt z1D8$;ETj2SnHk5b01`|{QI>cg_ z61QZ+N^ytDnhhr=0C}_G_+$ckXt3ZIsBeBJ+L2g#h@9btxv~_`)ClFy4)J+EJCleC zF+36{H?hj@gVn!}H64D(FJ&;)U8pVsEiDc<1#DntiOaG_7QUn1E2BjR1sKFd#4H3~ z7lB@n1>P`hRSvS6FScq$$b&q_sP?Sw`^i8XI-Woxaiy#Ay%a}Wh>U=$OcSxjyjP(V zWqkB)ijk95U8^i?Ue-5XDUqKi-<(=KzfTYRnwD0%TOOxEbBld}#oD*CkX6rNUSOD@ zt1EWicNAI}z;I_J?;1rIi@-bf=T>ZZ`w+d$xfYI5GCJ&;Z~4=Qflw{-nju{e_J#xY znXTXBk4f0p@2I%=0ihRkw`V%odzvft3@16>p+7fdS6eCUv)GKie_H9j_X1VGqVF(< z-=KcIKpK4pG4S?(zWpj_;S?Q=sQ(MounSYa4f6z?&o54H$tG~Gc?kxded2@K;iYl@ z_rpiX+h;fjP5VjScgKsYM*z(ylZOOjTl-IzZk3{=))h@b3Tr`N%IXYCR@ZF%W9Hl( zqq2Qcs>ZPElW?3$(X%0=u$Nl2%;^2>z{@Y+s26R7kZ5wJV+U=dVycKhJP8`fb2hvz z4*s5+HUMaB#j6oWBmb=#Xe0m0h)RoahNCohuE~OO<3plUxv7Q z#DZoD(WL0LSdrtSFg=IWlsdE80Q9{^YL_8;F5H9K4>nXgLhJ-z{@bH12%x6L3_DQS z(Ir{!QJpBt)e=KdlKh|Dv5xRfHIYI-7Mz)`-rDG4M)WRyY|KyW11*1&$TEm<$y6b z>(}E@gHK{d?^wMqUNQkRGpP9xVhu!pss2L`)Hxg%rGe+B+#HK*U_2Ug9?s$m=yq)# zm^Namb`OcctxsCN_1^I4J^qBi&Hh3P+6sJP8q}xFIdS*y-!7{-{$<4 z-Dc^WaH~0Zk2EWGc@tE%rSe4ZM0cVHdBIp6c3u`Md3=q>z|0i5DfhjAto=ZQG4ufQ z*tMuN-+e`%26SQ#Pu`DJNyWr~p-~+r!JuW7gDK>P@O%1(Q_0M`;ahx;!JpYt2_2cw z*Pc=uaF`;BBL!APY>wHoFkUN2QdISY--v#z1D`grP8NY6kqbXxv#w(qYr{1CAVAZs zw1uSh(s68LXxg_yzcjPdm*3dUORP9wsIArAY~^Qg7oX*7WSQwt{@Q;MUr3cj^#5X$ zs|LcEFHpr>$gk?mxc7Sn9^r=@W-UYO|pTg*lQ$MH5`T2HlUg9Z;$;v5rq zOe7Q4xoDP!UIoW_#&K(X-&=NJWgpsgd>@dJ6Aw(f?b#*3w=; zPub=hV4(Gml-j}kH|h%OUt>G+Z^APl({RzdpHJ-`>W}#eeSdzW^`VMkxX5ex57~C4 z!>Nu@q!ivi>pv&|6Qj+zwPoNXe2LV!3&uz(6_1i^#z-zJ$p5%W@erWDwkyeBRjf0g z_qA0Eq-I6R&Ev37sjsb0?GDxA_YHVv`nEw1NhOvrR=;CdW&uy%MU%BY3cL7-Wkb_{MAc)-b`$|NIza1zImWrB5?d%%rk`SY&vu|3IuqU|mK@8S%2!K-+x&*|lr#?SEmZ7$yUhTw*SAWYiF3y2sm%VIBS(Icd9 zF*(y^@{^N(iBAp@fJU#~A8oynA_yhOJd~Mh1U=(g%E zBCFLzft+Dv6iI;(i)4oybeIBvQLCz3&#q$A#qZ}?AN$g;kU3y7!0h873jMj!d{h1j zepbO1-#YiIU|ruw-n-dyiI%&6h_*^_+=KJsn85pN`@2M^=-etVIGk8wED=|6+Z8lv zpHXc4Lk6e}VccRaND~Py5^q>28El9&N|62}W7%`XRu1Dj5XHvLz+suli(3I7_o*@W zwZvnT5MhOUphP?-s&};xJqfI<9aKL zqWqUjjOkD_F8c+QYyl_UIoa{5Y`%w=E8Que+>iSxK2x!xxx+(1$XHGqY>rEO{>RXx__H*(tUjFS#ss;#e z4}Ih{ndNibT-r8%Zc2|CQ;%3i9EE~seA8{)>YoUVISIl%PZ2dbwUPH)3a0DDYF%RN zfnaz3&HL;VvG@^$11^U{)3s+*A-aq-ux`AYP*{Ggs3)foo!g?nZW5VkmE5hi+NGe{ zhu-FI!W21ktWILThFjGQ?hk!T+SB8v1Q`Lb9*LDo{T*_HKR_Y^(HXwEm85j3D-Eh# zJQe+~`RTMdE8;=B%Wa+ej5S_+~KGI&kxD6ckT4$y14Lmo8Tp zpUekwQt|AN2WJjj3j9jlcEU1{Ue~=X(T|$!k>bvNXqe-I%ZBy@{S|*0|fhb z6&kI455{qt%qfU-%YA)WcjWi@=(*lw3URubI|3|gi)oc>M#Ff~W(x&@Z$4@t0vMql z$KpBKH-o1$bZZm|<=%T#d=BnFlCPuB=WAVXFZ5r5Pt}eik9FS2ghg~aJap%Xox4UnwI}_tLM_;h97ir+Aw7{pwq$(ZytBcQsLaAv)ywvw{ z{V7OW6zImk6lwVtrTCRc87_cY@P&G2;Q`}{32Z3>-|um1oSkcm*!9Osi?2TEqx47I_F`)CA%~J%ACA$#NdVUkm_Q8 zglg72-V~zmo%pOGKHTp=kW8@Fu&eDPJ~HPs9E%afHE-79lN$K(0B%HgD}P;9NO~k< zfO}TF)ogHW^v=2f28UAA`HypGkuVB%s}Pa<{$wdQ zmsNIfJmJ{J&y>I=O3-Ap(?V%@%lW#GtT1W(`=|2L3hPLM2D1S&I?o#39F%uS`-gWN zL?VDuD-6*?&yKD0gk=wI{usdI%}{`FJ};7EsmKRRcZ%l9@uw*0c8or~)`xwA2B<2}C;-@tm zqvK$dV{z{3^ov(r1)dh_Z$ErC(ja@hNPQ(+tXqgchAIkVv*lLDXvZPwTJsZht5%-X z4p^5<%aagIPl_-!Ve-y(n)iFej*Mc&v1i*#!HR6s5Ec5RwCG~ZWRPBc0FkoDeDS|$_hqA{}!;kPMvFX9~TK8!^iI6P8IJoEQMN{$)#>0LJlgtJlc zNpxwnY5h@mNy};_q-QUq_|69>fl%5Sc_*ghCTXl$l$Id@n>{~#acC@l^wv$M%49O} z*hypRbu14v=XN1})Au=i?LSkt3)=d>RptSVgWCxFmGcA5Y_EYy3LgQ`>2$Xjfp-`- zQcD?-xbHg@UV$6(5gj7K?xO1#k6xFaNm}RYuY(aNOHO`~>**08#>~w?CuRI0MG)KW zKY%X=BfymQF)>y)%guP9sTat#jlYDt=K*G3yw5gA>3r3Ox3;0%vEUit+qF53a1BV&9Fa(Yow-K>3qoVwP-^#@h>a=CSGHoiAqw-X82W=e)xq zG}%br1n1hSF7Y9JVnQC?aK5RRjct>(;_c(T{TrtdC*xRf_Zqx}iqeGqLNI2=Adc`!637`~`MFmOax?})7%ob;P_*_S<`H+rT z0*ZHm83%|Z)kUHMYnh$h|(R(GZqkx*EQorXn_9|ZHn0=R-lvI;4 zWw=miG&4r#yV zT7djNO+fjyQoCGC^dNX;zhK3{9+!opPMeLc!%MUfM!Oo&XPul#RTy&1brZN9ke)|-s#~Nnd_U>%@l0aNDq`JK0!UbiiqOmhG&HI; z9LyZ+06$m(RL?incauz65!Yz4<3NM?M|5XNM%XKhz%tP-VksN*BZ?4M5kkVHMmhMH zqTk&BZRycYSWu~^Ztd$quBlAPX+R!+%Y&3;gej|A7y11DGn;IDQQxp(J-cdR zc2SSCIafC!yQ5^3l2tb`{p9r1uA_Hc^#1&uqkek6p+{)BevDNNu4@<_7cayAW*LqA zYZksIS4~EKJp|I=^0=<8;r`Fb^3PoNebJfXXT+_;Yb`Y ze4(woahn5tm|}2Jc52T{^dxOegR9OQ61hv)_|mZO@S0+(AZ4O)qWi_YX-IK0?(d#T zpUI2xA{|C40mPXk;q5!%C6LJy1<9hN=mUuUaRfeg_m*q>hffEx5QVh0RI&4%G8MT> zpoj5^W#1;oPI9__38H|G_%3TX=*;nTl?zaa(Ed@)eO8H3R4Ds<(PX^M*S|~d2Tp_} zy0KI-MO|_pphjLU=zg>s1vsk0EXxBzYq2hBro2U+Pnz;Y;Mnwwr+1n3CYRKk1E!H^ zo>zWq{46p0WxvW9wkAPHoR~}Cvq1sn$cleMvL5}S2=jZ@YU-~V5MsUnN+MSeP$3Z! ze8bEt00mLh0DQl1_3@!~89YUENmzf0DI$;7*1b!^QDk_))$$k9e3ykaS z$z~M}TSB3cvpj7Re>>Dz@VqcR6u>3V+^yPKffAJb(Ny-l>~F9K-IsYRVp=)8V?G&z zLq)tcy>Qwo*Nx+ntF<^XJl{s)h!tU56`UUOm_S%E;}h6m{gdAY>noD(K+2T>1Z}F# zOkL^Gr0?o?(Ordw$!~+MW?JPoDJ zoNVteHYXW)v=cis_!vq#XaS(~a*QKLv$W;<08!PUz1qBap`&CQwLpbT?K^IyFAO=$X5PuI{$mH8Fd6Rk!P^&q!J> zIX*T(OySA}g3F&Z&9jh{tw!+8X9Jk18oYXgvDV7ismxLfK4NLh_nx`JCcIK0deyN?!qY{Ud{V5VGGrw zT9VSKlY2>01(H}Qr>eac!$-6zt55dw+*A?ZGHlx$)YX{0@_3(OVH?%f`NCX+p-OKC-bN zz&j@BSYEt0g$uG~l);-Xc~F@K>~hiM88+ALK`GOC0Ope(bbn6fV*MR|B^nwJ5m}sX zzH{7=x;A>OSb(M?Tq@h@*4yKkZBZ)nWcfws86tm*qTwLovp7FYWi)%;`kAG!GEk)1 z9eT;9kDtU4y3~;MEG7qt5%%vI&8)VLFT2On;q~vvNRiXd_%p360+aBOQ~CqDUt{FN zacDhgTrQnN=Qxw0LekCn%eh1irJ^`c8v&wb1ecnbKwd-VWk{SE zyMy(#tRGXTM{HQH2O9|>U^hIxyV8d4tR%xYR9og`?e-yxLI5cu$xoCYg7#uX{y8xF zTL-=_&)Ol}$@8s_gNdj}cGJP__q$nfODXfk66#~lH&zv2me14LLnRT-ELkXGS9nmN z-u*wEGkvYu{pes>NeBJTl#hQlA8MR0`p+PfQ!=KZIQO~ny{u>2T+kyH?JUE@0|Gf z43vN@XjRXVoch&2t9WafR-(cZ#rs@Y(^@!so#B)5<@g$R{muLT1&iFM%H~;|#M$^i z8RFvCz1J^aTJ{zla*a;GsM6WxC)Dfc6@eA;%JAi@R^Wj)B?_o5H!Zq_F*M^)LgiE< zRZupHJZ5Qv5VmZlKAM4>i6!OwNvZgbBPnTRXHn!Te1)8&5rl{-)r&!c33hVpPa%Us z=}Vgc3IBX00j~6%w&(~`3A6Kl*CT;`DkyaKVzKp@z4iWN`zwET!zV{!1l$%+L^5bo zrIU2-W#-CGVrs*Qgw)ZliN{*R7tB^}y7VGByj}W{t7jjr5{8%_O*X@8K zPl%fsAr|g$J!(t23;$X3cc%CL0-Dil6haH?6b@PNNfyjN{LVcPgALN$AMSNl*%Zc00LQ8<*cV3B0{pt5+LV?)MO@y6uIaUXw}>7Dlk<0_X@y zUP7@!Q1G52aR?ez#alxc_mb#-oQQ^Mza>)i}ztOQ)J8fkAm^W zH!;;Rhfe(2-_IC?cKj-S;~ENhuW6@P>m*#tX3#29d$CPlr`1Xpl`MMnZo4w^C=?Ps z2)|tnVr@j!>7IlNBfh5dO!#*HxY=e4Ny!#R9seNiQtJBKTL7D~fca2IIDsWRpQ_X< z9?DTy_z|xAUabJGNWR;+%w%#Qv($ADP3_QZB$QXZzivhprPZcza;o9yD*1<-eiIv;-UsD77>A^+F zYz^`RDa=??YfZJ4xbMd{!`6~jU_-*NfjdgjTRZ7;2@M2K^`$1B#M3-Da`K94&w`q; zSORoAnK#Dzy(Ud-nQbE%zFiPkKRg0-crmdcqrQ$`>UopZ*%5K+$UcdNUS(YfZpv>? z_pgwyycysYB&8vnL|F8L?m4F4belJrA_&23J2-VXtH4p0S&m`AkfN`7Zg8YjF9`7q zM{F;TB?wGFdOCYEkmHcpytM?CB7u9N3Ev44XFsr?_f`&)+8Y}sQC}mRX$eM1Rl3h5 zua%R)P@7aj9R0`VShnEykXdpx3>&oIpz&Bo%j(IWG`ESFE6`E9X63QGt+^P`lG=g5 zqFRAG!RVS<6O2doc+Z*e!1Nhb>#{M_=yphZ=5etjbH}sDN&GW+zJYy3>RS-1&kh{F zBY>z=1DdJh0a4;Mrv_gZH(cIjq`!&`(hi3zBoQRIbn5$Ky%v{(DogGEgqR zfJ4G&?H1{nD=8eR6QArvJ5bLc1Wni(j4KVNOzntyQxSeQ7ws`>!EKxJ=|(!ZZ6n6T z2Dh82DH{Nv0%M7xKxe6h+w78c&kLbGXp_IXXRH2NkEelA4aG>cVRy7X+-l?k2cAZ! zl3}XRUcn@dDIT7pZuP{-H5v3L&?KJ!;MT#9Yq6BcSVm7QHjTxrYcarFifU+TEP?UT zLboPpZm_%+7cKONe;P04lC3};SVw4L0hO|xtHL~**l%hs&@oAFFnRQU<#KEcsyB@w6aDp|l zD)K~Gf;eRTwUe_8%gN91O))H%myTHpVA5N*x*)N{YF0Wf*2+WL7>Z+U_=|F^pZ*5`9PuUMfN zMxKOi9o}d*JKO4vI%0a(X-MZ(5lxZSMhIP4t(Q!C%a`*{DW%%uiZC+4`r$EeI0B5$ zEUsS-$dD1w9&9SU)XLKcNW4JrT9^SdDPd&lZrx9mQ32K>Jy)N546-crB?Pj}r_j=Z zi_FD=QJcKGKb$lj2O3%}4{8{cV(8e)j1U%^v~p@N5bJ1S9o{wzhYKq8G99yIVb&_S zGakNQS!RDR>a}XQx}%gaDa_z_xn2RvY`gyI$*Sk`DNv)Q{(S2KMb~-sVan>R5Chz@ za{WG}dCPWbA#DD#SWLE;;AzlQJznZFcq3uny{U9U$uX>>76$r!_`(-PY1RD8R96Fu zfWl1wEqCOWp%Bq#$EwaZIFax#uURgO{EYf*lpK{pM@_d9zkbrJc z8nyWum4UX|wUF$%h}kkdImI=;>OxQE$^Bmdy#(}2VbFFRLgyfi(vD*JzU_aaQIPEKpSl@lsY zorqVcScCUzib{!WAL#F_lQr2%>yDe{A&g>SgLIcsimmWl5FG34dyp)lt(0iKPJErZ z5F}>~-@c_J{CX-(=A+GO3Bvw9(Mv z)Ctj13ps|GNb*=h>uL9?-uT!>M}JQ;Qt~`q+ihAPk4A{=+KrGF-@ujAkIk^GFdY9^3gs~FyvD!_GR)@wzYRnk;!dCUF zj(_z5j!pJ0RuucpZHuK1_OH8msH@#AQ=wUK7VMI-1KO~>&qkdYM9gK8m020x)=ICw z8T*!XhguA?U>ug;xFoHj>t1E&I5txA5p_!JGN#jjLgfFh*mF*%Q?XJr60N3YX~*T_ z+9k$~U2OzT-;#__a`G;7vZM@T5}G@9{-wyWd(f0T(i$>1KiMMu6KHDbZ5^v^IF}FI z3kCSQtel?}(oBcFt{C0G8?TlSDm@nk0jiWKmRjc``0^ZJ3`BWJG0d~xD-yJt`cb|b z&U5@5W6Sl{dH=7Eg?8nZTEnZus{q;b#hE!5-Ffz55ve8@e5!#C1&W9psZ|CBWa-PJNGw)@wZK zI-)2D>_Wd>$d#P)dSEz+uXa8CmnrL}%8j_@<_S?POrz{-tV|#Re~E0%m-R~ZwVsW~ zL>{0+{$S(qm|HRn)%PSWtFtk2>5 zLmNF#>BhNi@nz?rs>`3cNgf%xDBnSQWbgcFbvHZTj+Z7T(AWelXQ4RxNeoK`_(&fW z!{~oF0_$S`RNV)qFmlUTz71ST1UVQ_0XwBIgT-85CQD^Qc$SmkmyrU*psE7$AgH!8 z%KF(WHZ+CSV*yBbo!Un@^m>gbm7%v0&T_#69@CU{HL=OUymCQcjLr!rDg6zUIGLORD{cw=V{R&+E*|A;;|B z(bl?g`f8a^`F(rj2%>JrMwVtg4gdpPCLt~$T}1wbnpsSXiuKu8_EVK)!meJ>7gcRW zE%Uhu9FWG^RDnFH=@~|)dmM~D!Et$#VX-kn#RUiL47sD;;-~~f0YRkBQ)dC1Uzj%XLPLZaSopvdRG!^*a6VO`z z1x)4&qMRy_yUm&-MKgQ}WV|*VE>1ER_j<@Y7LjUYo5T8O zBD)zwtQVi83QB4M-uBK`Gz)?xl6mTOQup;Z!}YG?D`FrxsX~;NB&xN|gTrd65;R2QsE;^Bl90I@8n<*IdFi z@_MPwZ?g`D)L101Yr>YC&at+tZIdhQ2sZBZCpFiodXQyboX?MxmW2J9=xFQPiky;S z{N-D<5xXroUk+Yg&bQN!7&z2&# zqU28H9O=)8N&7G;4%BUHz`1Ymst02NXX$-hyTYV*#(J0?{>$Q4p~@cbcg^YhI_0mj z*vwMp2YGF~R9XZnZVoaS!C`YsMe%(x&Uc$@C}{rm`Mp`xcnquP;kj`lS( zgUTB|Cz9(yRo)?ovV6HnkLQ;yoLwZJxW~;6vj~%vTMdw0^qX!dH=a!&I`ZeCj6(E) zwN9DfhQ`h*LM=}mz!SOeqDDfY>V?(@P6sg8^*=m<@#KUqd8xLl|BI}9imq%47)2f1 zw%M`mbZpznif!Ar-LY+_W7|$TcGB_5-upl2-gECuxoWw6J}3M3^js!+DJ|DHnEGIslXNE-3|T>;?k&PL#W^t*loxQab@ z3ZZT$j7k&V98Wt2P(F8Ir@ov9%UqhPL3%i$SU_zkbv_E>9!11uNHK{IsU2gEV!bAH zbmvr$f`|-nSbVcLwAUexiA619^WIMt^fSDm;la!4^qDB+iEJ=)4lr7!WCU4suC{J) zy`O0_ASLyy4LVgwx?}bvJ%MekYScjahZ=vo(rv>pC`{A(4%c^Di6r_o@J}lDR2NFI z^n^K{z}!UMgRGR8KB9)g1YQdh(Wj&n zPx}~P|37$qEfFo_4@kA-B2Ihq&CW&EdaiK_cfN{V@_q9pZ0cmy{%0Q%uOz4NQi?P3|r~6Ge1Abm%}a*^NI3%=CfA2UFHz}+9X^{{7YU)jm-%yaswrG)B!f%<>{I#&G=*^Vb0 ziqxjNl2EGTgBi9$$$EqR{~&>xN7NRBd`u3BVTzxm<7a9UQuM7mL^o<8tag|euVpim zGm7>#vvYgj2WF-QP>u)3oqbr@@Js%K1MaLrt@v4+vH5Z)&qb8U6d7AV+2fADBx95N zYT7bi=#sloLWr{!XX6ld2y%%E6OA7-8%4l3xPEd_$oIUjhGK>%Xuy{9uk#i`tFA%( z!P*Zge=C=49~%_Hv@2^sW>Qpm!OH=rz|IV<6UEj|S|IPw(82+5eqWxFVnO=cV(@f{ zB`(@Wph5^*y+?f8@L>{6RaL&#`X%6Xy^q&!a|!)LL$y^cpICI|F*HVTZ)zu%@qLQL zjL42e%X7^Lk8si-TT!w;r2OJ1&#&iNdj!G2Eizy!IoY%F?IM_M=>szI=lW5LcAL@oOZO> zM|otrPndXFOQ#B6UCux{b1L=rl;}z~!DjE2sfiWw1EDT`M~Mo-AuZ_!$j!U!~0R}tUffUty(`yX&n^8kC)zom9&Msi?MjsJDb zLv|YNeN~Zfgx=bjn?dk?i|F_FgYbA&Ry+d6(e?G}VPV*fftc%unJOWkHu&C@;%0%( zNAQd{67~OpgXvM8OBtHR>gTRJKH?xNYuRpg2wDK_SI2a0J!_e z9UE8Yw{?4NYI+k#KBXlO(-!T(mimdb6N&z+q0Z+tSuAJ!*nF^iTma!~XLAjk~oM&MGUWzQ});x9au;9{zJuz+|h9&wkPV#STexniR?qynokE1Eny>X|;? zNS9rkItiskGvOVqkIHkeFa(ZH{l_R3Qk<6aa$hDx+b^^EU{eV!M*&GxzL{Ew8L@7) zGd8xfS3URi>PB7f=SX zxDt@3O3uSX3h>w)XBQRC$|7b8AY{zRzg<2ERDj!&8%y;wO1G6J7!;1O#SnD^8My zYIm|{GSs%`8Sq$@UnSCw>iuxGWr0F?!wr^q&QSd{6lk`!lorzp!`F(%&pDD!W`(lI8^_EPo^__uuK zTO0hBapC=c+AjHiP8COti9g~#DT3=e#1c>`W2e=-AV9e0YUIW1v@rX|Z*~cZU*ijc zw(j2PRGpOGvbmVS!Iu_oc7mx8FZ>?8)_k6WnH|A0!usFT;O!r35Q^h)$ z>zf+vFi(V$hnP2Z0}>IHzNta$H#NZirUtFDw@gO!@#Xy6aD*0gcn;^o8BB$%4K4}# zHPR)^%(%S9)%TC7u(|5(Yc((x2V0AuPbJ>mj75f$SQqJ`Q1aM)97?(Mi)QRKr|>Gm zm5ISjWwl2SGe~ngGv@@$t5-9jYU2wmzF)kYmBbEasu;Jt59f#{415GqA;twdo1<`M z{`=C$%-fJ6G06Apu5_e=&2+3Zf7pyjc=o8;#yW#3^%-H1!ga}*rI4F~u-J$pTQXg% zp}gixX#SxFnAkR!b^Xzw`03*lY|=FX8sF65E)ue~O;0__gXEslENSmc+r;sftW>IT zUIIQ9Fohx#61BuDA2)~zm74GbtLYJOVz5&O{n_$q;NI*3TR#R@Tl8S?7IFAL(7@;) zXfW~(4RZg122KA14PJeFu@Y#0KV$jm$KG(oenSJ#Z)ouLf{`(c$*gN>`5m8LuB|03_bQl-V zk89eEdkIMv5U!ns@{D_aFXE?XziRy9lb7=jadd3)6lv=0 zgzV~RBQ$o|)~di@BMw=6dQ<)9B4oMxBs%s<7+f9tiMgeLGD0+EJ8CTcLYM1z{7HS( zt(%ARp%RNwY@7YL!v*w0%&?(*(mI_@kN49e@~g`fxWTT1rGOfi^o5}rmb8hcsf{vR zCSd=PaRz5}959>b`K+9Pl2udB_7cSEw-yrNxY#mWw$6mx@JFWyPYv(M^a8t@NQ{%t zN|(tSKh9C(6E=ziId;rX23gGg;CJCYk5 z1|=X>HPuGp%bL8_P!6Jt*_-H$w@{5LaivqtnOZ7>V~FUd_Q6J_Hb`Nlkud7|xQUXt zX~~fNL;Tig)Y=UXNOD?(%wLj?c(y6aO?Rmpkyl=J0ZL4nq8F6f8<|LSxO5}qiyDK7 zf-IBh$O@?Eg-vx_&6JwF`*p$f#i*k-NfXVqe)v@^xa4-d9iI2;y&I%j1_Vb{53Ki} z)OPNX%j&Jc3gh zMa74fBnxI%4Lp|}4X;^Fp(yV{A6b4KL@zMuXMF^Tp-oZ@hVB>@j>a{DrRTwnqQzcP z9UG0bWY@CnZztS_&{<-Opeb0*2SUQd!K6OB$sUusmK7g>*_8DUE)eJvFlk0Nc%4kV zK%4$LLjSc}&2GYTN!NcJlD(`-O-ypL)EOmjC-?RAD*Kh{C7aY7he!hjo(s}tBx{FN zP>k}Pm+}~fzr+gpEe)_bg~nhbXJdc7gXphnfRg`p8U+#VDegy_=vY9cH8kUKr)~BxaJ}pIdc~Aj zScpz?lQwFKyFKP3jbmnTAO~86FaG)UrdHw#vC9mSMZLW#m8Q~&E-x`Xj!p``G?sbj zEP|q$s>(!Ek&&P%dvZvlnYfwb{9Kts%8HZ)xuslH?cRK=MGcB=#!JSc_<8G=*&xF(PQY{+fJ_iB{DlF=x0cgpv-EH z*)L?}LGI$;*(GqB;*1b~S?kd|+0kj5y8g1H*L;+H1X+Za|9{Emdwx0{TntV(FFSTN?2E zrM1Cw3(%gSUu>3vQz_UFG-1%OEDy#4u0 zmVdi_T+29|iir9(?qPh7aP?my&&>BwH}6kFS=991=l^y49^l`8|BlstkQ1O?Z4&u4 zG;R7jNmiwJ5>h)+dQDyucxTyDnDOpWPpF-+%l7r#8VIIePR1AWG-TB`)SvfvItT@Z z`DXj^!O)%4&8X@;vNSX!G*K~6FJ|j-#v5+loPkS5o&_J$rXBrnYk;%OhadFiztc{S zyVgwA0Ka`ITOo4zw|yg$db3(vPP|5&zuZm9uWo;z0j=P~E`Dz%v;6{47l%3u=$&Ie zktEFOQzpceeO5aD`AaiOb62JSt6y~z*xS8Jn`Ah#whHeJjAnf5tbVe~(i-s}YcL+> zV{Oy=CGAd#B8oF20EW4w%i|z-Z5RP0Ke6c(8Z6J~Jti?o1QDJ7O~~K1piqL%hCLPF z0)e~Mp)&7>=$gb@_UF+!<0(fg#bxo}0rioB8B#^@ov>0z{wTzo}H~QTJ zpq|Z79wC)u5@ux5;pxFiW&SuCs~zc-V44|f*+J?w-}29J`kmAm7e|cPg=8)A#oa78 z0PmgPcTR7ao-ng{<5g&hp-Wb-;5MhSEDT#I2$k8|ig}87n6ZnfP_{0V475aN#UZ7|;v2Zt`V0DC=&3?HHeLl~P z?0oR@e4fwEA<6^%K5yN`2~Tba_3^$Q{{o)lcJK)a@jq^+HfkAW^uI>q{62+LUf4d* zPl^av&k0E*@Cd&id*NNrftMSeWIrI(CSdOw0i307`P6edYQn~hygGjZlJ@Ampj2N} zYphVb>Z~uOn&kaR-xhx}U0tYDF6_9Ka@&4ugDnVk(PJIlt+8;F?VrrnQXAjeVC`EQ zRMeL_={7ehDu({64QTAY4_e5lV$M#;fQ?uVi& z*vJ}t8M4x|MwY+ZDhP%qD@u;oPpz5`ted-n3 z7Rzb`Yx*#bTa{Wit;DjUGIEw5Pdxpd(?BSXKOXtTifV;pI6*G`5M_AV9EHK%%Z91J z-d)Z|0*)4sn2cYlDJIoY!F)!;&=MxA%XFuQ<x)&K-kR!>=E6|#Jt}9L2(dz(jjW-j2IcE=GSd^;c z>0YYj;u>T;NqWNFYp;ep=TgeHvUBr=m=4(5kEe(ND_d=ghqDiiXH(T9lW4JcGOR^r zrh24GRUu!=N>eg-T{$1JbNYyS5~u6AhCL4HOc~<`N>3pek{Y^sfxG z>{@bXusI17u{#i;@fBgAXc9lg!{g(x4@;$+SJCC*;Lbv4wxPeFw6)xozmO9cRe@Mp z@ffpn|E+e>EM*RXBHC5?rzs4iN{X7rl%9>06(06%8Xwm67y1!Mfi#S`M#0R=s2w=n z6b$FrC6Vr~FDx}n);#pa*3(4EE6j^z-G+P#`Oj2r-H@QEYE7%oNzDVLtE}XBfw}PA zV2~s`ytB`KKcQUw!R?aB#fPw%#F@Cg(OzD@FXvl*%se^$Zay#ZZOJK}@+qx#3Cp*u zco~!2btc9%ON4*00cLrZYETW_A(@%bDo~=3Ff0r?Dk#jhTvg{6v5t(zD;b?>0RA^N z$fb6)Pp)l&f$Y$s)_61RtXGu;9|}UyzFa@IIA@#&onX441$er1g?ii9TRL;o%fGS8 zYOZJI#ZUYpkYr-O_m}~eoZivT{+MB+L+zw~2E<3turl&EEq3|Vt%5I1^7Be|^l0vb zIti#0#_TT761B^$WoL_n8#ebofL|mc-?z`=sbI)&Lj_L)9SXumi~{teGs_9+QH?X| zR#yAM&p+&>*9(o!yZ;-o8g#qA zE6(b9*_VSxipOW&3ik2N>ch}51xfv=&Xz0oF#SS6K2OI%8!m}vKGWA?jIea`z@|Xp z0uh)|@pwh$|Gk=v zSQ}6nkwj{D8agX8hLy3{;nAwfRQZ*WhpALsgf3uoso%*e-(@~Y!5JE*Sj_fUB`T4e z)|3Q#-2X8Z3j<=rF6{#XZ=l7KGK4FA4)=WWXXf(oSGNeXoMYuJ955eBUHJf?26f#q ztx(Ru(rbxU-Q*T^2$7orLxJPZgKZqN z?c@wVMp5j?aH8n(um^Qa1=kruT^>6E>rf09wl`c?qilGp9tzIVsERJd`WGto0ubeD zkBkm1U0zm!r61OnCE44iS;C-MHeZ8$#w?uu$Y7LY3#Rki0R|2TaHV0nO2^q*K5xd@=ASjI>FBPC^ie?G-4}{4p@j^|tDwq3#|H&1|}KVjMBn@?vaT?6%O8>29ciVK~Mkv4{t@2rm%T!W8!I=w{gQ$y&mDy}4(COs>v_Ke8q z;N_$n7qx3p>+2Q(`*qnhU(c&=)>P3nUW`8kSoO^u5G+%^I)~E^WYLs*ZUax%Q(DX^GMfeFz~KS`xqR?YwLi z8CE}w*t%e031nns$JE4M1k0?Mb+JTnW-b>8h7DjN8`v~bRyLOhU5;Ut?rzO8b1NTc zZ|*`6^G^sT`Q&H4*K!F+BNhgp07&uW?YFrbmH!L4)3AKZ9AP zPD*$zg`kyUIh_&6Lw$AalGxr)$htW0(M96JoE4LdDT!0^Oa;Y3Hmg$^dUpREh(4U0 ztW;Kxv=|cCNKETt;@SE%XCGrc&vIZ&DtJd(#~!VsI!P)bdgPP$NwUj4aL)b)veu@_ zsTlDjHh2B<@14NHQ;<{U9U0WJ&B)`}W;Bo2ZcsLoB|mtc&*3vFE@V_)16jCL*G|2x zDjHZ+*VcUODjE*>lvD8CzuV}}sF;@ahAF4yl#fqOe^-C^fcvie&c^zD*MFxz-M-_0 z-Y9nGzH?iE|3>lnE(D#ilkfWO;QU{QX%*A|D*t;b_cnI4m6Qj=Z(TT*aQIIsuAK+`sDRDpjV6ykP|AiYEA7Kn-d1fl zKOebkYr>$78dNgQ6WL8>SOn#2{KPJujtp!5Cb|s2{kH4jcBox~rdxjxEbWzd2lM;M zjLD$Wq}ovf*}N(&tb+Xfcr{N;uamBQ84d%>z5_xnJ?bB6;2MqTaURYgI)zU>4_sm1 zp_!abHkjseD-L*xzf^FBzaD){qT8pIg;TdV?~HK0H*#h^i-T6s&VAxW5mUohl- zzQmc-GqqYV{3KSxd$n44t8t4!6JB*hx8Ln@QD+}&Kpgc$r1DnX3@{k28hLqq>YQ6zfbCM; z+7=h{%Sktx9b-!i79Q~&@KY}DBhqXwkYXk6E=H&PdEZ5IykYA(8mFeGX6O+~#do1v zrlx9TIpl|%EY&gIhO|jMVk_j-m3vVv%z%uaBKY0LC1Ai6G|R&0^I<|@kBPW}%+C|% zw(JZH8|6mD$?%wUB03eFwZRe^cBi@iy1Uk86C)Obnt|bKVl?JMN0BC#PM%SuaCs47 z4~4sF^P3vD_Mcw)^lpw;jjz$T4jM}st@m2m&_;~Go-J|XJ~$z+$;t{4+rje zRRmg!7YOK^8kEeKS_%Y;L@`$43v21%`LeIA(Y(_}vPT>iH#(&CPg6 zHi+G#Yq3*DNBlv-iUZ;*5T#BXYn$gMtI%yCy%!_M=b5#k$8NG_8j_vqT{v%n7 z^u7Pu;1dC^t&UY=9S8`B44nITQ?#Rkzh4VHcyk`U;8&F>w>YejO^rwA(y5zLT}P^s zzh&n-1(s1e(aiDpMaj?DF{uKE5mA*;p)F&U$s#D9Bu2-s856tifxCtv>(DXZ+|s!n=N8t^p22rnMhP zP=xK9u77KoCb7LxS#)+UM; z14Hl(zeY1=!3GR4vG30pK3j{f)bYF z7Ep2d9dV7EwP64^!O!^-*I4*d+)NSs6Jku|?`W>nMi}iJ!oKC$;dex@% zM^2+-Nn#?#a;n;a0$W0h1eB!*n$M%Y6hvE)yQ4)T)k+mmsm!#a##1R7K!u8gj2-f4 z+e}Al#q!gPbOuYeSqjP5i@@mA{Y%=H&u0^*P%*Kjfm=apS6pu8jGT!So266 zL696M`SObjD+4n-r5L{;rPz;s?lS{{)|H|GCE= zCqvnmDIEjnbPi(t``wa1xv29$Z|6#&1_+;c#s9R_P?j6>Zu7rxQbI=~14Uq_<0$;V zdGM?~Pd_)1T(UsQNFEM!%`7p`Oa*cdYB%)N{M&dha^$;%e-h7|HMXSA2|sw?PO9$O zV_-2UfDF3&K7pG8;L^KWsxryA7;9ajQZ;FocIw}kx?b_PX; z*EYc1?zs>X)y0Bn4AE0+SYoYssn@{*_3`KLJsq7Jq<~1}N4jMF(KsF2Bn_^@3NFDy zX1Tf^vV{5*-kf;$$8AMf>GhBNhgBjATrp4?45@&1=RokrrQLPxhd=psJFW_kWhpZ1 zzs$2aU9xwdOYJ1aBBO1?A=#BNi!Kr^a2N};FfCr7zy2oUwXgaxOvc<0Pmg|TDpE1s zj+DCe@pE}TK_)sWT9j}*ckBt`Y45&)zZ}qS;O+ZxVG?4=zM)F42c|`7q3{qZNuPog zm+j52g_3pGp@oilkPFGa<@1b7k-gx*b3x!GL4 zP;>n-Im0ArA#|=c7Y|hUnVjtZ8d^s>dtyuqu}th4I9sLBNyJz5*EQZ+xMj5Q<^F`7 zV1MoXCC&BD739qTRL{I(tzn!#-FHK;jvjd`?hkGVscd0YqgqLVmQa-r2~a+Ddb8?X~(`R7){Gm-|u%$2#kk3NGeGe zNA%L$Rbv3CKCOLF`^;p|%&%Ro)Nz1HeC?Wy=t^uLU3UmLv2y#F1HZvMw}frX!7W8ie#)8BwA)F`j-nJ0M2lv2$R z9oN?t`5in~6qmhf957h-UJWk~)(Ri}W{vP4%%@UKh-&98%3tjlyHe68BA6q~Y`1TE zyIp|$6TcxW0RcfgiS)eVLzytUP?oo=`-N}gUso)y~_8KO%9>AmUDW;?HT zvsSSIorOXJ4SL|^I{h{x`p;!tlG~OA@`wQG$do7CU8NA8_=~|G-Cr4l&!rtBHD;fz97_`b;x4F&W z@15{_@(o2~gUV{5-MiL6U`i<`)WzxUDpspq?}*8P>()EBtco`6BYKBI&@Y`O{AYspLjnPd_33`T?1YzR=;O*~C(#1%7d-|Z+qDhZtq z%`yPRo3hXLQ>)s{mlP@HpI7 z5NZN~Rhtc-8q?H0?RwtKfQNI;(-d-hUxY5m(3`MqrjRZg**T}T9Eaod3RhZ&G5A>}r7 z#b-iWG1vhh2sL0dI`OkHhZCOO+i-6O|0xO%=uk0ATSG-*9)YbfB8T}#qU?NF~26cnG& zneY(YEhU5Dxb*rU$HVUhPeDt{$jYORoV)@*P$s6doS)Av=&_-&>QpDO5K^j-C_}XF zNx)PDg$xp|QV={zl~1P!Yl zO(i*USNB3aFlxe8dG9unwTgLr9)f=?K|QC{VbeQCyVG{}3TcE2tY@1FFneTHvQz zLN?JsnNkcnI6}&su2M;b-~TPq9N%R-?{~%yoHFAe=S5JQY-3Q->Q0 z`0pSb^I%%%DZ2e#kAX_Oc&P`MN?9;6drnGynU=IJZjms&+>Kgs1E*=n#-8dihqhS9 z@Eo6Ah9>gfshi)W$1L#@^9hCXn4VJ*scYN{%xtL9n^c)@M90Mlf}hl z-}lE+A%<(MLG7wopY9uq$;az6fk+AVvRs>nv~1rx?y_FkMwLf%kmb}zKjKJmg%krq zX>E$gPkId{S9+Wvo5#+S3wnwKuKE5|!sTm53O_<-n727`pm4*B*ji@;w$-Ii*zKSfff(=|%< zNjltZDDPZaSa3lSM|rPBnCk?a+jMb$`|R1^xu}8;#x)Pip8YPimH8)F1t@Wh7TZQ& z?)+Q8zXUI{6$w+qYu5`e;*XTjbo;b2(EeqXCZjR3Ds-qyH))(`B8~kgPV)Q+8d>bV z=3aGF$xR)yWqSkjFqJNp@)U_Cd2W-t+47H5I0!PeCN>-FjrDn^Df@i86&Yl!wL{%G zHWrqFq9gR9BTI$Kj`_Q4^L5gYw_ur)H+k0$e8!U)=u;B|rghcKp1pL;<)e^bgJ4mW z>!e2XE6xM|1CyS*XV1o4U8h_UKjSHA692A1694R*%qvjFLqPpU)%iCkq+^72+CoN zcRdpa(%x?_E`%pTxP;VHH-WbODPZ=9j2;Hu8FJ=L=I-kAlnJ{yQp^iG&N+NN8FFRJ zaDRHongZza*EibMrhb|2r0Ik`RSGoAdH$}2HacrDGl>qWT$&6k5y1qPR`{~dja<&Q zihy^^^&RqP{N>iQ#BM$7(&_g!*g#wTpq2VwDs*`EpY5Q!5v} z>GadDLdrE$&!x=9q0ub2*d(4-^sQzmwp~Oy2tAJu38LabZqTKt((o1{@nQ29C{M*H zoLPsur_E>1gJTer2)SVh08o}z7Ru&N$0$Qr-2;ldUf2LRO2X+6&eh~xqbYcUuSm-Y zhqIkA4uCYL`D>dS*Zk?A&EAQ|HCq?5dXh8pq(I!Il-m4|`#!yesco}b9TL3{y8RS0Y5M%&iF#| za!RU9ot(4i9s%q^j7WbPKD$v=_LKHF<@5(9q#bv*(>=zBRf_yQ>e4dd<-)5Oi>~d_ zaS}o_r1~3HOZ+)%9udSQ+41Z2I3pp{`)_l}+)XEETNr`EISChN$(+7a^aa|Wx!(#( ziGym7YE^bSV|hP%T@zI6_x<$R80OE_9vIoUk&Bt;|I3w;fZ8SY1!wO zB6i&Az3#`K+jy_bKR|&iISbxl^?t=$@2i+yhZ-Qm@=M%XT4CzL+slsNe=S05?C{2Q zcp=nY0BUQp2?ypJlxXn^^NU?F1#_O&Yr;W0Xg90R8Ki)CiaoSoQBfqF=#e}>z&%{ z`KXFBn}H@spgRJ}@^Ww_uL`8ZkYf}%##XuFRC&g_Rc6W@xEM6DUnRxL^2zQ(DUd6u zBGkplx>c3TgP}d+CdPc1b>#j+IzFB0Ke_(+K8X#DVf5>B@AP8E`Dx+w!f)zW;Pa~2 z$E;zi%7qW1=MTo0VWQDZiWYhFg2UIb-cCci-FX)k^3#)#1Y6nVl(wZ!QT>zj@LR)A zOqFtN&EJ)cVd_^&LjA+2G9VBZYR?4|pS;e_6rc|#{=Td4vJii^TL{L0bg9)sT`w_u z9r7+>b~po3d+TS0>?nK}*a~-ed!b=NcJT*@BAYuEFo>?%J@uxdY(pfoNVL~VEx*5>R4OjKBHU-L;T zyas)I+}*ucOEyDJOmCPalZNaMcS$eY4zm8!eiMw0E zT$qJg?GL`0bH0)b&HlI%!p!$=hy|^SCz<^ADs@%|>g9WQzmEj!*Dd9$u!Y=M=Ga2wo75c%R?GE>5elO zlMyJX62v*-e5bWe8b(0Jq2vNj_zW7~BXK6c4eS^KotflSz!^PrFB^JlJZo>G+eyR9 zLprrCC1jf2rwEr|J^hiGPGo*=Aulf0*PZ+AKCYK%8}F<6^IH45SNXbkW!vv$+mGh! z`s{D*hzNVR&k!w~WRcCXQ@g50;jqZ3W_@H1QcaF(%NM`kg-egMupyI{s7AJ9+gpK2 zv!)Oj)YzdV8yTjfkVytfmi!CKPm?~$u-Ni9ryPJ`4ND!6c~E4Q{!IrzqQ>|Y=E}>? zGuBGGG2nhs^a*ntd(YzXsR2U9vYQLyFHxUjdn$sIa+2m?P0P)dN+8xjG>+2!uh0mv~Pw-T#cnF`RhFp2xX7FRt?-8vF4aiQ7 zsgvJQB}kzWeDDIwP(eQYsM}~uqSk$@MPN}OCc-92NngfyelnkNe%N1-Kk3-YFccP3<&5a?PMC6gDs4Jt#V#qeIRyx-C0IQ3Mud`k$;~62=w+K z)vH&XCQNTjaRk}Ti-rt2H=bq{d?iI;13z9{qB-opn@|#NUsT<<5^krfYBT3l1)cRg zf@gRcn6r}{{2?(Ti{3^K4~UBN%UWXb)JHW}9Pga!i)>Om=sxN>#OpbGE(528qD>;* zME=sh=TA*i5mg`PAnJf`=ssV$2@Pu_#89C;hozWTk1oN+yalHA+xVOZ&pUpIJyEs2 zOkn@=wbu@cyGwPH*w<5;Ag3YBhRv7_q%dYK)>}v1WXvoaMGz8{-e~XdN2_WwO~JIb zILy)(OD_3EVL=@=0F&$C?yqCsf@4lw>gHE8)~rJ>ZW7Qg--20b(y|L{vL+ofEZVrOMtlD?p>InYR;S!mlxI%c-^pd@ zlH=~bf^1%cP6T#ne}Sx&(A}r8dxh$n81lZ>>U7}K`KcjRz(QoZJOS>H+#^i;4Gurk zAz`Fbqt6uh!O^M5UM#C68%m6Z@%W)bw|N)m2++dqS<`p{4QDwsEswxEA^285ZJUqe zM%uAKCWw1$#QEHV;TpYtM(#+8K5hkU@~*6)xRiYBpq$W%ew48Be89dT1ay|w2;{Zz z!s1#fi+~765qPUG$ckzYKlcvEdYHz1pU`|3yADJuXef(nQK^y%1ZK#|g2}ru zxcykrZC&z+zw{pZkwZhLE^_8!ZwQ7&a-KQ={c4+ds5QFrNv!-+MM(O{Z?(SRS=`nA zwGMl~GK=A$R=GGvyM#yXzAqi!3%#IX*lBS$#)C9WR*4F>aR zrCp&^1rD8l5%k2wY6yEoH|`f;y=N{m6d;K9(XuxFdPrL~XQm8D;p3^!mPk$50R~NF z`ao+Y+hAtn(xY(V{kPs5ZlVc1Cw<*EhL&rZbC9j`6qnoAqp+2Hmhvc1m_xyu}M(X2;o9qo65p}q=zYU@waLZoI7M< z=xRQ(XCxm2ukyt$+U5!0Kap40y-fW`kVwwz!V}9d!P|^oG{^8T!Gfau&WTEG4(i1b zwS?Vt!U{c}5vR7`1MbZAK8UUc_BEqVL|b|;TBHkH0Tp&+V>v4xPEm{5TWugCbX8Ix z_ZC!ApA$NZq zW+b`l7=4Wga01n>ZKb+Sfc99b{_{`GT`eI=Hd}Gs#wn3fd>T_oH_r{vVj^^B50`Hwp1bVrXEjJTYX`B=ohr2=(e!rU-k+eJ zfNh+{t~f)$iU}OUD_X1~%P#U+)RskS;;34vU6A0Bqbf-J&v! zev@_Ud!u1+@?_J_VG}$pv8w_T9Cxdgudw-Vsrz1ft+gq7^m6=tAX(hrxI95dOGp-L zDWsLjx|fBdXv>m^q~#%-k8^>G3*P~OXH1I7&IKwJfBa*ua+**;)F7mu?Na$5NTXI_ zV2ZTnv5C+Waq_^hRy|ME^VIMU=L{dT!?TS-15;u2&cL7To}_l?ej?iA4mBDDBg0Z( z1j6C9dMOR=A2X3=Edh2|m0w%7M1n4@@rx|DMHlFp@BS>@I=YNHJkxdi>BAWbh>CW) zENWnSFBI=mTH~`cwF9+Pf3>m_dX<}falK*X(-GdUyqop8-C<- zG-d2-zk@lmd+Nf@nH8;M23C#tQM;sJ^{F5D-0PXR9N`s}Eweo&I3W}42dh6}JyJ66 z8gp_j#p^wfFp!!tC5MXNP+&1d5Uy5?n@g_hDl(f6c?zp1!~VigZB9tNa+XBJPmVwa z&Zg;#2n7`pN4o=-%VI%h1h$jrd;}n5bg|Ln3=c>)9$p=rQt!!WZ)EV`xz859E2c*P z+{g;R5AUj^G|@FAQnnRxGi)$dc@h>%HH4%qOu3Bi=r=j&Yp}G5VN2|=QRBI#*ic!! z$mw^1KuX~!r7qlg14%=jHCaG;uf{VFZdQ${xH}J|!Y=D1fNmT;pi|MQq$;%T=MO*G zxODo*Do*enZb<}1ZRhe!E8%)bu4 zC&GEY??&=2WBWH2+0F129&j5$h?6m^_ScE!KR{ik35;vPhXWAdpnt%-zpor2f`nmG zhrl?A-F7x~ITKYQ=Diy6-m1o;>JxHVoWqbfn`>o4SYq7k$M%b@IGt->6Ti(ytK|4| z2gN*b0MrsGA0BhoPFO1uZ)_JX`HpClxA)9aL-~&A2`NlLoQgZAyKD$FKmudz6oHX4 zbGCAiLbls{e4QU}cMtlPZd)2wv0(Vu_+3t8Kqu^a z&vw(aQPmGv+8BtQvni~i7(kWPOi<*J@zt7dWqHHW*{{m<5;n7MNDgSpqhdbLnTRtZ z6P4Vg&fvp`+GWrsrOaw-pR)NjatP#|KR)hr$Ida-r9&|t%wP39Zj2u};Ol^i=I^tu zj)Ff1%?vv)=9<;bo++?eZppBZe9{io43XSXt2|`S(g)xTAno@rCO)X&U2Seb$vo!K zfch(VajW1LqQ|OMjylv?nB1C(SZS6QQ(uL&vt_{`fj}s0yM%dmAvb~+Xp@38k1=F| zs>-J7d#zz$9Dcgzs^k2vZs$Syag#cNcE_LCVEwzq6YDX5vf@8KNM=m zW_stwjb zQ(Jd!_=oaVcP226rm9$PKNnI=6~e(ZrkmdN8S#H!ooIqSI6gQaeBBP;>c{!He*N9L zz5U*$ywmmZdiVYq_x|^UP&7sVpx2lGHK6;GDRK1nR{ss`ZrRzX0ML-V2c?Ik`?P!8 z?9*U(qt`D$uYrp{5vDL6CFRwsyrX3D{Xd1YE>>^7tRF4Ur(v(e&0>SQlWhHBvxnML z<-TmzcdGTV2bo&7ZN$>&>8wuEenm;#%ZmThRX&Lv-K6qq;dB@*d2?p3$r*u>sd|#} zzl1#xZ{#8MbnduO9E|(UN}p8AzEq>q8nd%Xu_nj(jS~BOLHBu@YG`Ik zHEcMMiW=_s*PE|!Uk6>-Iz2iZlDhpsC`>(Sd@C#`$)4xmj&TCiqaSubA)MV zO%)6ssD0EMKi*qHTApC`kdh9pYz{Z4JM=z_0B!78!CzPyr=83jWOLZDEfZz}eVMcJ zTcDmREfwr`%d_L)G=M*3BVKtDdquAWZ(YVFE^UEaku`#3vXj&7kc4`q@R?sTIB4CABeiWyU^j^Yg`;=mxiy%PC256Gk|&6eQB`4!>C zQG-wUm7q2COv_Q5qk3g3&F=m)pMYi|=FNp>*}LyBr3heqf6v#TWi#jj=TLY#Jt?qfT5+?yk#R#%8}{$A<;NrLC5m8N?)`xMQ)y5i6&I9euBJ1UgzQ4=He?iow-68L)@jfQ|8ls7Ww^(sRk)LjuJmvff`yYELe1LoZn2B7Csk(J%#aYO`UZFM zBOmywD&);ZzxPStWdVMh>ox8ruiU^Bc$-A~{{pi>Ouy~mi`QNV=NBz`tuC4iq-^9n z3fUcl>=5pTL^-P{w&l1EvO6CO*?jv^80Kj6HH|u&u(ue_>W1H$QjH%JGTnLE{Zz_D zE%{mmCTyf*hOatWFx$Zl>!ic$+l0+SIJv;s`4qR>pt+~M0U%_V300M>kwN-eIaF3N z?Vt#d4GN>hxpo3h$y6bQ;tQT}U7J8EW%OKWB&!>q_NFP*TeA9z7X?fE$BohFhUk@t zM7E)MHedbJHJ?yH7A~j-Thkkkx8N745oQvqS^5Pmk_+W7SuSrNM-8Q`H4N<0U#j{n_OwA8(pm<>OoLzfKFnrkAC!uq5 zf25c>S=1OG9b}IH>@AlH4pqje()NRrrN>SwtAt|u#BZQd=F02b&aQyD&0j{CtQuWq z7sB?>^QRr6b1#E$dp|E)i_w-&+%(lNF#-2`omgfTC1<+iiJFs(dW!4g&R&o&$kqFT ziMROHCA-s4=TGO)y3`%g@$$3z)A`dY^Fz}o7{u1!9N{mox;&(P5WX|AMYZ#tuKvfo z2z?Kp$WRt!U0XpIavo;g%%>uc#A?$EVg56?{N-00rMh z&kUZ^bD63+c}wjM1E$+r|uIg5opD>;Hi0gKapR|Z{G-)T&hw2&v4C>{G_dHM#f5r9Yr4*Xo1vB>B zZsMaN4002TaU|}NjTnaD9NinUts58-hpbaxW?mjt}=~gGk z`goV>>rz%dc?Vk{^9bWXGq_?cJV%Dmi57n>CY`)N&OiXkLZ)+aA#-LoA}o(WP#hTS zH`Rh=5S!FT5p(WAT)j1fjnXYG5Q!rSmR|F-MMq<%;XbMVqae?4hs#?C@cR1fRc(AT z-+pxg(?*`-TZ-uwPssvIj6F1sl7A%FEkuXaKzeS++itox>WbG2hl6(N~ADm{lbfXUX3@>L(I3OOt zk^7>-%^TSBJyiQng1_g7V266`6=fw(qdC6eEQaxEX}+_Cadf%C(m2kaP~TZaDHGO@ zlku%lk4hsz=FJ7hEjOBzw{$~jrlcD?7u)yi+|2d>bLE08q+ah_%VrsOFG|m_*GJQ> zX-8k6qJT1LVZu_QN7^7gs>Aq(_hEtl!aowPgRSvS!WoFz9A~?rGR;;1GpN}!AYck9 zW~J=i?3H3AT7v=DY3Bm60ql73@|><1q1x@FzFSPT+=pC~RDy4{?$)2w(oC@v!^D6`H39fNRNbXtR}hFk4FeFV_dX5H zzVD|TIt@b>yZ-oI*n!_F*5iWNE!ed;-t@|oufYx6I{Ucia;7O{ydlqWFZ9CA#jjxB zS`A!dkt^8IYTwR|;Af9?kv$SWQ|t{F)!lp8Ffoo~P-tJnM_9GtPFuAB^_tY`H9qoi zzf|CDk&~^>=q^Dx_Y~oi8KI=4B9(a$$?cw*^#Q1GZ+)gq^kJ81*uf2oePk8AH(&(d zs$M#@3ZScdOJ5#x#`$}v-~Shbm@J1+1G1b+?uEu26FwlsLx7JU>lWkjuIm8E7bP!+YF zH+av+Dh$vxoVlqeQf{PTjIIriX6Ogb9`XmdH4Quc<8QXDHR)=v}cI@QY`EMdtPq@^bTBe zky@urE z1_mg8D_}#JzRrw3woAdyOn3$Qcy! zTJu7Mi=k|3Fqzt@ME3=f zwY`D(CU^lo7jf+lP8}_by%=6rkZz#h5_oVb(L}X4Do5KAljE0tz9N9EuFYH-twgep z7NG{zyZtO?$6VGv6H2aGDOg4dTGE_pjME>i5cD|ZcL2WLlD0TyLl;_nXl-;iz5@f# z{`>KuYC5w)6Tkz*IqjAjXUF*-KXNxrQajZ5W6N;@lsTLr1o;vr0D$h+l$lohrpL?Q7Z<& z6Pj9o&UFP~fFvRUyL6gCQ24)p* zBPKiB`2g;P>{Lb{ur{~#hpgf2aGwbGFFOj*Ie1pM%h>04P|hi1dKg&UQntb)5{7=EU9QH?kT;Ht8p7(VRorcT$@ zz=--)nO9rkVQm-Iu+eDPxz*)ShzUAtp8MTwfy|)$+-{F#1s(X&X6&9OiL8Wc01} zb0*B3{;p05tWyH(?lUE@_C8YrE142l$M$bZV4V_J_m{xh2^$m2(7b4UH2{?3$!#5j zyf;uzjLXOKzH7~8)|&zSU# z{plHNTBbqd7r%OY3ugkpHfxh|Ws~I{-hIcQq|kD1g)PG@ctX_`$*W9r^zJVlEpd!CyLvf?vh-*hr(cV2Ba> zs7Q=R!gb9Ib1EvDWg7xQ>ztek(-GBuqIx=j%`px!s}}YW-Z6WOi^`?K(;Lb&y68Mt z0Dx0F5$nFr$)zO!tXR1rt4uCTPftMVySAJ)Buwa%W?oqDM?o&k*t#czt0r^K1)^H#y3<&$K7FETYL4=!PghQJ&6%dNK3Gj%HI@mUI<# zG2vTvXcero{o5`k&{*$Bie-r2%1Y>qYH?2WIwoKKRjs|fAPPStl`>uG(BmjsR=O(t z?t_wLj4C#m9^LQb1##pRkI6thd~M(T4zkiOP)o$aXL7`>G85|pJ4-mY~Xx*`V$Hq1U}R1Ar*&b?~m-473I zd=}duFDt^?zk-tFktK0#7~@-(Lk}OeW`QLmabm9GKe0DF=L6-jsMx$;_WwMK2*UN> z3(Qq}qtxAN-|!UBh`_EP6PmdVGq3?QsaBbjTEW@<=f1%*6; z!fb|W4$E3^l1mU(?|5G2mfJ~T-x;83Pe$FjTbIcIk}nrP4&x&n&LD6hH%~){rH-e7 zXgMdTswxM87<&(6hoH4~_~ekjRm1#j9XY`>|I9TejV#bpFqhi6`Cs6ec!*^mfA{mO zK^r^zIEVlE;`pz}Kf8bY?)doV=!b9KetmiV`bU2J^ADoRukl|$AN{~y{}@T+`1r58 zIA-o?iRN7BnJi~`a9%)|(|ApQG(am+x1W3Y;nyE1|DXjfn5Is<66E?%iaeV?e+E18 z>r?ZmXY*%&n3Jz$NiuoMYAdel3xx$R2P1Gp7(Ehv_Gh?bYxqSb=pu7f{Jv&_l|0!R zT@Upoqec=TE2`POw_!c&v-+-GVJBugKi&88?GO}VGts{H;*`zuqG|*t87k5l&7@%D z{JWRK#yfBofW5r|F0$E(64n~o0P@|goi)r1SiWG8kK0S1xN0M5UAc|!P{&E3EH zoGI>ss&yIDFzOW+!Gldyr)NO-{lKg^s#kw8Lk0l6*TLrx`XS4LVF0AS|Ay^RhEga>Fs zMkUPC5#Sfx(Y(mm3DHuLOo|nvtDXvUP?J7*9g^>CS zx19qWbsrj~^hWZOR0@odf>Z*11!)bzwIMlIMrt3jJ6>uWK z5$4x_f)#4NK9nuyo-cMJqNC2UaUA0*#;2CY(GpJ$eWT)1ROw5ZWvqmWvWdqr3%~YL zGQggXp))6q55g%Ag55??bkY?YM6pkE^8MPHMoz^BC-IfHQ9R8qf+n|G zq`7{*?N;~I-7c2Z2I3s>Y3O`lOY|GI{#%m@sE>wmykEP!LuudT?%CA7gYOfP?1Qg*wof>JiIg4RewdvZyi#!&}4E z>os)r1oNQlptAVO9jY6EjBKO8tOgHUr0FS%x!UXI8^%SB1Qb>%-MGQQ)}vfRG=W5F z>A^P%crIKgxbP4jT0P4(0=ue`gt6Mx=D16nVMlJiCRSca7s!!ijoHdBsRS7?-3S`q zm82~^?!w61*geF0szieEV?1h0*m3R%>|i^vi0^!fiNsN=Bypu`-NraUfTNC7d0U$4 ztKJAM|5?!F`WC`(5}6lN+W`3!l%;&ZM;spNs8nqCaIr1|0R zx<~IxK@p*`_*|l|6!I@h#t6@ix_W10HIkC4a%gp~ijN7Vt*SPsg$dW5=LCNM0L@pTk?hw6G5LTA5#H(?;n{fiLr&SFvJ;pi2 zPigm3yI;WoXV$dIsgIe}zcEdZ$+rq8w!Rl?Piv319YNaH$E_E`D%^8&bea`wuU${` zjG+u&l*_I8X!ZGJDV3^`4EEdoNpGF>)=6(|KmW>lt0&pqbBMK6)HbqtgT3bw^+Mf2 zj-{g*DM4GHrv*{qceivi7YNtH#mZU1Ke$%EF6kiZk^fkI5ZtdeTF;((I|x0x#|Q%m zmFB4T^`Wx9ZS#AQutG-~)``D;=H!B*iqRe}No7(2ux}s+!+#7PPMM}WQ_Y=s*?r4K zk{rI*#u~v1PN#Rb{aadn+#ww8L##ja!=C-9&iaK&XH1s^@oY&xFi6`>wl})cf z9OMNfYra}LatSSH!Zn75lxeJjj!6?#SE>;oLG}PA{2raO^@{IXO2jDMxC>}Z7U0ge z!=d5rCpJ*L`>W{tmwcWPz3t~-@x9wmo4>rl$Zh=N!m2hd038G7WAEW5*u5&<|$XEkCaV4y$v+!?q7CzWj-#!iYrhWKnsI@=5acyhS z%A?G=c0fC{RfIveib3O9yNoo#%99kkzD4%m^=o0{<4m8me3J9Fg> ztUaUyBPci~MXg}TDK@%E9b{Kgm z7F}qE>KLAH$9JE7IVPCeJ@Mu=e1_19cHa#^;%q-76}PeD4Wb=wKM&FMMz7#lbfjqA zu7#l#Kud}w8o*TQL{Ew&9fz@0qbWrzDbbiqkrc;5A8tR7rBOUA3H+{$ zq?fVJm7U`VvTfWNh?KZEBWcNRSlLSWkflu>9p|~I^Q~#!-Q8-5C4$sO23mFw-%1Q? zD9Yo)wlpcKvXCmS&s6N39x&F}TqBoShuNKR=Kz)g67*b}=V=>G#=1Bf?2H)av+UUc zTcu)ccwBThG5z-N5aT*LFOX3u{K{c*yR|h;NEk-I6~**?t0G&Zs@j`0iC`iNSqZy` z(_o=m_YBXZ8Mn)&t{c7?!jouCYmmuB=XQ`*Y=p3iK_$}*M>EH>TTiKLR@+A)Xb z8D&-Ip3DbGz`LG73YhivxpB&yRHPpp3 zzv3BF8>Jy;Ifi78OcBGow<_Zz0|d0m{ZHgQq=x20Cx2PHZWEXiXW#E)nGpJ3@6Sj{ zA$rtNl5SXOoxy7K{u~LL+xtQ^noet3UU`U%=ij}oD;aL#B@Ke-6GRblfJ)YG&8Xr` zX>v<94xElU5ekjGu2^Re$#&R(liq=`f{x;r0h(q~tQ1fCh}N(hH4@3zR_ZYZhfTk< z;NKzcQ!Jd|>G_%cq6K_u(W6*ORI+9{MIB|3Bh&^`gkz9PuzuU5pbJ(Sk}l#Vk>ZAx z8iGq!g8wvlM`I6aC=f89YjFQURyK%wm<4P^e~Zggi7496RM>IHt+;j}d5O&PN^re- z0USz4Br0j-ePZVFxXql|JW{J5+p3s8NV1NP)zVkediB^f=1jC`N?R9nWth6l= z*Nfx=!auyexEx$mD+~Zrvr&ePDZE)&Gxc07F+_N1o78AbOr%iP(OyH`=oPA3KqQ6> zJs?`n$;*&?)bI&}-+@}0FKN!QmsE9wLhN%{Ap&}4mgQj!8^)1oE$(ZJ>Gjr?rmPJ% zb+O8lW$XrVjRq!;ikj93yU_htR-EI>1F{*uobO{pjJ_?mc$^2WgWiS8f*n;x61rxa z!7APE$!P(aY>o2dpAK3BELr8op+BQjpKSr0+ip< zjjb>0L==TF@qob!w{@daE87A9dWT7m+zYsNd)WrzREYLEp>fcO1~~&`hsowOV6xacppMPHoB>q3%8eD-m3Un0AC2bd7jL3&WL59=0EX zp}8f^&g-m@qU1R(dA0$e3W_!@WobtijKb$B%L}PNd4et5Dr4o>hI$zgw%t=-=3{pk zhg#@M&Pw$ku%a;=EpXr?cH=N~**2hiBV;=StYl3~nrK#XrFnt~!eOP}ZVc3q07clW z?^>4owAgH$=RmC;nAS)kAr?;TF`&dgM;NgvPxwe%LCE_X3mf(rkYT?9T-aEsFdxey zBpJoPgtZei0~CW0VO;=V0h+_`U{T;-ffibZ)JWj3V5fHhtpcE50g5|-V|xPp8VmLX zP+k$+U~fQQBhVhB6Qv&~NqYYY(vOiGJti@FM9T9BJ9u2G^T;&kF)6+y`WlzGJThtd zQTiGe`4y2w8IeF49lvtl*p(4bS&spe^(YWoW8tx)2|o%n)+4}TJq9G!csQ&U*j0df zG!)h@&{at02*6-1|NdH+6p-cDH&nds~u3QkT?;5 zyy}sVLh{xC+*J>lRfwJdPz zMgmrOa(_?{4TDuZXwmwCs&-+hqL^z4q-qQrMm#`OjUO;dN7^4a)xD#FCOvD?vnD+& zwy%+JPmcrjGy>*nG*Wv!#8Z4?PZYdU8@MTgY5>})1GFiE=n$+^CtyA? z3QehdqiQ`Ps6D*wTnmU`Y1>LJFIM$AcinU;N|2kH!KY)zc8MNy4B< zv03cVuoh!rj68Uv2+AY~VYCBH6h-x6@I_;xi$=f}#X}a2n??d=y0LIYqcJYV!W6|q z6t!T7q6gdyPqYg;6istK&_tsrb{r&8G#t@L%q|jwBJwae;RT`)8W{fnxusFkL{~JkWHC=N-i6;R z3xfEp>0bHN=blYzgalY_2c&`gKD09TQF~jf`8~!dc^7WU62@g3UYn z`4#1vxvZfV5k>^Rk4Af5!uO`B-Hsbx8aiG?vohy`W|(PUy7*3}3_E8vkr!KY6xK(# zIszT#f~z&|qIw?dTRa%Gwya8Ewe{D&1WW{DX`PTxgiOs`2Pihf^!Cd3ZZOBf@Oj(= zF#~UptUaHt#U3@ChH87cUC%5g)YdWvXkM07D1dr_*NrwSY~s>(Z!!JMvaQfr;!DEr z3W^dp1%b@mC%KWw>gd9mFPSDZH2wwivcEddesPn`@z$I7+>SpuL+Ig;f!6*9`FPOJ zOFka_0?pyPKT1z6y(%ri?%eAm7C3BiNa)u6Ys))vkj@lwUD=>mR&&mm1WmW8@gb8c^HnH z6?dIbXzjCr4MH&ydL_c%6pw?DSzB#@IRLK>cxF{$#EnH3jJB5ERZw%5mQ;7LHJtPB zUWV*nFjxnk1~z0X;NO72>P3PoB@<4=(QvR{;_+Y4V|tt5r!wmC+&ba_5vkf?gUGggPz&byV=_XB&9>prF%$(y53pX>zSvX7!*h z=crga_jfO?{m>j@4ndcAkSCRMQaSgjayEG_g)GhfgUtFo7+SDkB%|fZ<;Qr&%KFN& z342x*VU81r_JS`{)QEa0YP-$#&q``~V#5i`Fw;U8C9c!%aW$8Wxre|!AA4ZAvXgkW z?4s$*R-)z$BqrQepHUwmWuiy(}h#u;dE!rC55>GYTVr1gKgH7V^a6 zG6>o9s#(*Plo-6cY7`Z_vz#l_jRM1VXl5ij=tPWjT-(4G1Zr&9>YxM~@`%*(X`BR# z(pPAQjTSY{83_;_DQcP~V|DOoQPT;N;nj*3H5*|vKy;+2xsSBR{fL@wb|)e6Hz*`} ze7BXKYx!F2y!2rUN?TPIO}j7 z=jD&JA>vs#!9f#cbWPE<1@T~Y-?jow4$)$Ap{Wst4u|J8%BR_yt zM13Q3RvL|cFv~Zgnu%l5Ju<5Fc_|m{lE>gzYz?*bbq%{={`(eiTo+N9XT(O-*%^un z`(4U1r}_mkH_Xg581c=hh-EV}v=N&w-FtG{Bw8~RTgAJc7o%`mVn$vxVy+P&(VU3E z%5Z^x7W0=grRZw>q}lJ>Os%V&ido5M3iTB1uTk^HMG7GMVzqUW%({4U88cm@hF&tN zhUx(Ij2u71hJfHee71YjT8rjEpI{C8+Ed-& zi3s9#K}b9(P#&RzP~}SGOt-SwPQbPwL6oX3WW?@cUMDd`Fgcuv#;x!FXUBAmDG`f@ z!wr89%`6#36P@>;bu%%St?P)h1+S@T2tug-fjJY@bR2K`MNFz;JJ)JXE+J4rW)`;@ z;%QE5;o-i;fdyl!vZ)AOZHISKvlrwYyS3+pMaIq%G@CUET+0d$zByp3%KZ|VYnEI` zSVRfri=f%J?u+3}OUg69KvcHbZ#iR4%N4t?KtZ;-Pk@@`N-+x-El*haBYC@FVUx02JDtjMfBPHgDD$ zKooK7>PW{bU$7DYB9f98of`xjv`o3qKLngj!&2UhY-5=f7hbVrSBtl6?Vr!T{KNby z9CyKPyKFAb!or3AIv%^CwWvtJ$B zPd`7Bc0PON?V4?xJXecEgtaGnN!FXkfw>?I%l~Ur++@Ds0y}4oQL<^Zm#7175GiyH zx^>@44#5+(X8L_398oC!&jP`90S|d7La+Z>(QIy5Pz`E_f9j=cKW?FLn9pu`mL{~s z5C~gTTuB>NW4KQ*MI-q+*3;|+TgS1WrRGVM(b5%j-Q0T}nr(VvB$2dNU1~GW?XR}) z@`9CIB4X^234ZIK9AR-Ew`b+j@tyimd#Hc|MEwY1bla!L9w)C)?40~<;}X{gLx}xo z7~4mCZ0r&>OkgQX$i_TMB^jzQ&Te?3kLTonu~ORGc11OBVzAtGu5Cqss>oAv1fTWy zGtXIylKHXA3C8^2z6~;bncE2sdd{KCQ9I{5SKXqSF*L$%Z3TOk4)V+n1S~v>9=eoQ zq&mbhYq-gOnEgjd$^L)#{{1;_+)5P3_vic-DCJg3%BowoXOiso-A%rVABi@$N7vHK z?5&;s#%L1Vjff2}1Sz>^v;X_2a1H>1Y!U>!C6ApUwra;|k%xF3ocCh|cMNaOo;NH) zltN_a0|`?x-A>65p`1v0`@LPWPQR6V;|~Av5xgcpuJU)sI}S zY0*de2MKm#5m7tZI0P4!S4y&kW*SBY)##$;AZ!OHs}*vQ$hOK$Vti*MOcZd+FmWmD z5U$l6Jpp6G;6<|xO^wB=+)&=@*^IGgj;Rs)IMqQPqAqAe@W$iR0v%li)K>T$ggdVO zXT0e;s^;h8RQnuL=W~ReEWWnqV^#GWQP0!Vql^H$cO}meTNhis^D*jn##ii&E{+#f ztFx}b838e(N@uJ3X3Rb4t|HWpV>A}K{$Hbc*hDar4b=( zjt!O@WY$3g1jwCG!yZ+8RFpJza*f?h-U{{wh7%84DXoN^NqJl4_DmtLQGbz(L~e}C zZS)w87NP2hw`@B@g%+Q$j}=n~n!1f!cu@4lE`M_AJjA26q8F^q^6*!Q`vzTLjc~P; z#e%0P6HwAnPk`|L^dD%y+d6oCV;Z{6y!`}v-ur=i#UR>Bk|~)0*^0)$f^`&Ns{IBS z(RaXZ2aQ2$PA-V3GRXZ2x4Z=n1MN+4)s3Ncr3pjXJosiRa$wdJ66}uYV^6sE}&1JI}hAoBvr` z5E>H*-BDodd<8KlXi?DD@JCiio3U2|Z9N!(t4PKm7=Q}$mW!A8e51(98GN05HTzKt zMv|;jB`f-m$)XWfch!jV6eR`NVOPu-=xT!y^qa-d3P=@scK+=A>2s}tpihV@cGht+ z`rA+E3WIJ>LvEVwM%cT8fHz!7CzZe4=gFT5ZimY9V7@=Zdvg= z*FG>WSi(|QT@&vyw~=7@v3?51%Q?A(y2rGxRtl`gdD!ZX*G~brpBQ&bzl3Fawe`gw zc|MJ=p6%_^*ti=UDVDnDJr6qVpi-8g^sXU*_b(6k7%lW3VFFz&ZDxnWYK+H2)q8 zfY~UC4Q6F)FoqCaNeoB?6{JSLK)Q>&vVv!4Hp$I+PRN3-DYV8150if{%$5hxRP!3Y zo(JeT=9;SJF+DLC1Rfl6-NHP-Z_`I-&T7a_AVr5#(SyvqgK0lI^L8OLQ}g_K%#j(BL{&hvQBoq(cvSpfk#BZA!n zR}2a7M$u-~ec=TG{lY55S4yf0={wwgP`;$1=|HI)H@F~aYjMQpb&X7p1;F!P3F~pG; z>^Yf{KS?2r=P30GD6j2US?@H0mMOoedy6qEh|34-#PfZjJKIKr~TH5<^{1E5rO*sAb$}1qIv(d!BRE*;Ca6(3ZuzB zDV|$g`g3GN?oIu?H4@!2;J#sdZ_y z0S0P2?`z@2E^$u2uOSddlQo8p;|G4%e{uA&9xbZmmX+3vRBZK!@~R{Yno*JHi_(s= z33MKUk0V5w73L=pYAt2(NiPPK3svf`gcLjKE%YK&@=TK(3wYNUhpnUF-~Cz*C@Y33 zGFIn;InS3l_ZH(1Rhw%Q);VGG6}Z$}sCCrrIkA?i6m-Lm`V~glRgHOg6Rp+Jw8{2% zN2F(3p!7Z%(io21o6Q+%aq{R4zxt!8#^!xBAdEI)`e;yqGePG6cY4cy90v_{^De&T)oEZKb5?LD-Rs0>FxYJ9*4BA6WVp%=_7^N;v@@&&NX4RH z6k1%})2>wNaExn`aVE-}-by-+jd=kNI|mQ@n9IT>sueV0YEG`J+*HsFrr~Kx_rtm@ z^AAK8f{7FOX)&|13oyd(1&5GHN`e_W{(qz@`fCmdc+Z%v( zvt~%yRCD(<%6wS;$n!n+M!&Yt1~8H4l8c^P&t|5lc@zVMj7J_30Wxc@UGhX;$UC0) zsx({}%O3exEd&4V`M>I2=uA`vBkvm~R^|FRdAiR6*NmpDFO6hO#Nwc!+RpPdp;r)# zXp}g(NmSWM4Yv&dhQ4}#^+Mm9Ki%A1*}@e)$Tbje$Aw}^Rm9x=%Oy#<0`=ZlE6QmV>xhy)Gy~={c(Yf~_%SS|f_HHkNoS@k$Mq^WZ;iKzl4T~TDV+Xcr{2HU-oH2A zzm@;HI{p1=_tp*N`emw5NZHw_n>W|Qh$W?jO9<9fWy%j3rrf>!LHd@^$eyAvG@K=< z(u7v!S{C|l0BM8i+_8c$p?EEn>JLS!CpD-*xOQCD4hm{-X+9#s>4kmA22DeGOj=UA zzi};nOht{KF89d97uO-Y51RfW#+T7Ke&?0HF`ahH(S2Cu9V_lQyFb5|g_d|{njOxN zyQy<%YjOU|n64mt+n(nQqa`co7Y0bk=R+jSt;A(|QU6*nnWMZCfh-`=V3{BT>6Z0Yl&HrHCbd{bmONv-vSnDRLaFmXg-Yw#xW482cVK^vT-QtVaoCJgU(^IzG$2XF zV9qVu9=6d~E8?@*=UcYD9^hsJoUChI!$R@k+~fG%Adh<_{S9J zwv=SWL|t#zgMVGpQeU@gR+4{qaZd1B2V_mCA@7D6o|=1va_?))nf{b0$n}QY4!GV;x|P&~Ti)?LDrAgR{C_ zyG4@}1L-mSxO^6UIrvNe%{-Wb&%HKq6(SQ z7446#Y1db4y;PTE#lZQELGuD)W8kS!T_;<Tw@ z5m{W)(_2$+` zo!(vq*8B%koI&vo&uEctLvi@mxDtqFXX1_*5^^iZ9W5{~z~~(bQ{;c0e0TBwFYhki zzB<(ur57f%tF8N>_%ruD(>|bn)NLJ{owq&#piOoin$m5mvZ7hGCF~=lG2ZQaPX~hO zEf?zitEMwu0?&RC^0@pV*#?PHtW;xxcFfXLPM1}gS7k%?1YO89%e9}YThtl3rvu`y zBxJrmEYtWBOxIkEOmmP((=3>*&n|Thcm;Oyu(i>N_&FoHdMCc9aiXM8WtC@EYBIKIL4fnc&!6gaV#-c1^kfI&Z z703>bx>qckdOtKol3?<@9vujJxg-)xOrDu)m{@s$EB8ROIf&yK$7;8BXS}Om{CFn$rv>) zMGc|ZVhPyOU^~s25;`QRj1Gy=F~|;b*f(s$o}w~|6H8en%$T9^k>KtqR+c9Bu4yOv zgan>M8)Xst<;(bevTtDjX#Tf~k;|8sz zo&H3d7Nk5wX6%{U)1cfPuw&C7tAc5{Budb6Pq&2|{X2O{ONufMP@qr_v15o@d|5#A z9C*bOQVx(O5+M7lw=x|eRCq60lKm$=!x#8F*=T6wofIq_Jn){;6xxNJ;Xl7Gc*%m) zD2fGX6d&r`+gOl&!7j%4aY5jWe!%deFtfL;*j7*H zKr;vBo;53W1Sb1puTXTL{4&goEyWE!@a_A{gLri)Xl9pJ>!&+;HT>b46?Z&g0}6w& zQ9T~J-XEFLFBO?=`0FT+UcX}bq3fq_qj3C=a=}F-78rvcxiGW{VCypumCA}F5kP6i zt+7b)^aab;f+_wSMyE!%eB0pm2y@oHsfW8Ge$H@i;T01X)l%s z+tn*JWvu_-S~Y-d(2Jb?k(FA=u^0qg~rXIiA$_Z zU=z+O#fmw8`g7UbAw~z2eb0L8u=H>3d$#P_S@W~=2Zsp`N?^?#KPwJ&ZZ)=>{OR{L zHU}prKjm|d=6#fS-jSSUFr{L@gbICEn)QeJlhl?&2PpD<9y7aQodGtkivSQn?C5yyVc(Q%~t&_ zo8i}Id0OxP=i@kT8s)+UBKBxTIqQiVWttnBECP#h`5t_3tS4%Mj78Y-jR8Q4Fb-pU#xif^JyJinNWjLCaBv+-NDGRutu>m25xaKm4w#2<#fDJZ6;Z-qr z*U<2BPFnA!CIFj2WWUMU=wBARbOa~t4rC>EX~p*ATCzz@rz~LVULU4;U8-`xCY6-7 ziG^$S`sB%zC)kMbx4-=@kp-b7WeMN39}EIU2)};%bWZ-?#oIR!{+O2v8#&`w^#cvS zfp&X({QLDc@9LIT%{M`7blceo91c(OH(!4}Cog#ch1I#~I%6X;ZOE{YC85yX55%Of zZo9@h?BrhsNHFCnc7HMbaq;n&S`~^@nqCVjwK|jAZHL71kgC{CirMa9Zb7>6`ypKRPEc)BGKnl2Z&jWWHVX5z%eazN+@=9M%VYr;~m zMv6_>$_x2dK!LC$Ss3evRaTS{x^2S}&sAXC$!H-Y)UtqJuMdE#R|v6mt)#t-D~+Ad z*K*Gp*OA&9^A+M!C3YZV|GcmP4e2#Zl-ic>UL-gqGnm$lwg^ z3{%gUDTO_4gD^L=xYatMp~!Q{I#I6ZM8Psz@;me{ZRpm|<{F&q@qB)aBt#ZOm9#K- zUjH7F0U`|ct&+RBMlodVw$c-ARl#*;f?6*W@PTm@# zjF*cpl+3CUfyW@j+a^t~r9PHH4>2Ux?lGX^5n$`yJxG*?NWNKKETMa-%0)9+YlLff zP}7ca4G)<-K3{XkgPS^{PCBNnNn_9K3pVLt|B`cv;yim~)XAkuhqO`>?5B|&j= z^1>zD*#M$TQL=x^AE*qEThc7ND_2h&SBXm;Qc<0SWv=lJ5;TDnZe>*% zpO>xvlE^~q++8`*|E`xY z_{yp5Lrrms^EPzY{uL97gIv<8xTB9@Xn)!lMnXJ{J$Gc@zk=Se&bt_ z`|ps1W|?Z&!?0vOETe0Xpu#RJrg4pgc1Iv|1Eq>Nwj*ZaohXWIBqpB`K{moU~s0muXRW1wyeOjsrrbX){_ z+-%Z^riI~?38HR9f&|QSU$Hio_u`%w>BUuV{#a8atDpCUQu{N}ztaV@#m5|mfWvFo zJN8q0!MY4=rVMu5lNCgXPA^x;d`Hv(7u&C4gwz>vaL<@5z!(t|^*eD{GXN|bLqr0aAy1N}C+)~6M zOsM!>bL^JFbEa~QEOp`Ph*bD`)DBp<*0{5eu-rHUH39Kz8GVY;h4#q-4L-=8Zo~ok z!dt7bfjF?Y3S zxDSty=ksF>{HPU%s$9Ht{y0zaDu!m5r#EaPi|tw6uRY1D23+&R zX$-wA7)h$4V4}>nk0^7wof}GKuSfGxYIhl!PaRJOT5QsiV)l@Du1swGE$+x2E!6Oe z$#^j<<#$}kD0B(-p>L0vzVvshCgq5jlhPjn)yIg~gT-5$GkA0fJ2rmh(?zEY0s@!WOCFiR zXrFE-E|hGO%YqhLk{Z+4c0~OPuK>ouMgL%aDK8GmV%)HlSDO)qG~juENA96WttozM zV=C~muoKr5HK)lfYDsQ8v9XGmXM1#H{NNi#hirjpiiSW4nKHfK77)&RghxL<5?us<|H$5G87JtB8sd`iyfK<1wi5pAG;O3xDM8OPU z#!ARwVXkaHM$c^Iokq?d1o>8>qqW~XvwTkIX(KySq53KqOEEttV-9>#33Wymr#3{e zAFx~!p%nzzy#cLA1=AZ0b!-Z`Axh9(ttIrjOK46LUIv=V6*RfkuY%7Grguh?Qxo3o zA6qlUr(74ZTJ_6s)V-Z#RN39_gln{9MbfRH!>PL%SEsl%-XE9;q2$VyUPgb@i;VIO zIZ=#}R~Du5FV|sl8Lg2;GgeHx_E7?Bfq6uE$mebf@arAuoaCuPtIj|7qeB64ShlwS z6RGs#Ixem*m9Nd?oiPQfgNx6kSo!s~>Km%rfgoIHJv^;whKv9w-wX1^^(BJkt93cM zXW-dIo?g=$J4xOSQ0>2E3K&gk?0jgA*Gs&oTd137;vpL$CIgDK!N@KQfMs65g-xgc zVG@ku`e;>m^EDj1Yt)F~6$*DhBTH0tgI)AG8nxkwk>BrX{JCM9JfmfNEXUFJp4Hs* zzUn(oMLMIIM!T!;UJOMFxSQGt#k#R%Rk*H4T2LExM+qVT+bmd`*1c!YlyWavVQ9BUqO%-jAa9*Tw#p8OZfnVhb4IS<{(VHPZsWnf7J`uZ~kufuU@x<-+j;= zr_c{Xv=@dYzPU&IS9cjh=ubb9UQ= zmP`x=EZb~YAxqO#cM0w(y(cGNEyUsnXQY%8=nl}rAj}Iv*hj9Q5(@4u91X=_b0?ta zPijoCg>S-$Dh&$#O33n84)tviAxIpO&^yU(Un@-l1Q_fGw!ob7VoNq$mGqX)$+Z?Z z{LlqFNkO1SKTAm^N*<0r(r>!&!#4Or%H@)%D$z_aFWAhALqo{lMNm!Apj?_LcKxL_ z#mKHa9b7z|4Hf1}wqv0&sCqmh6?N+RDkkhe^@)CA29ZbCvJu~7kdGB$!0LN_Ey)>N!m zoyxtTDd5PTodz&u&9J9Ims-N+B^=8_3nb3t^@DIVP0xnA9osGBSuf)uBm5K_R??J~ zbW~*aTk1MznTQwIFfS4&E@#te0y5{2R^n|KoHOhZ)KV(vOuh|17q4QrIS zuOYEyDM>4k#$(#j=t>}B zH@AG)Fs*6!ij}Gk&P0Bu38Rr-jO7NH9UQ|TRBDVB-HHJoqLPL$Y3@6;_%SXz><9N& zrg8qfR_nI<&$hUfho=4bF-&0E|G|c zLOVu!+G6rm?a(ypI@&GNs{G^LHKg9eZMevwi1t)=Fs@0^bMmW(fMbMVr_p0V&WTm= z=jK3c?WpGCF%E0e`XESV-CC+i9Ex)$TFGBa;su!0ZOWuZ=t5>0D`2~f_8*4^jVdP4 zaRpspXU_cNR29MQqhMP3j4G@qY2y;W>}8`8?Kk5bhxNR$K{CL&5L?{Lmmbof!6ypp zN-s;RJc2GV&CosMbMtNTy|_J<9;EkdX)HDVPfhqyP;1`XfRD_#0etQPS)%|O$=<6M zY0}y1cv#;~6&J!#?##-sgwwf_h=`7{F&z*|3-MraPq(Pi#nmOeZ1utw{140Rqn*Td zESM$-b;TsuX#X3dL-cSo@9>|YZ-;TbxfbmO(_e}P2`imLn?NeICP=Sb^CF$)v?#Y| zqd9Z4j|S|gO-8;nIwoYOm5@Ig=M7tDJ!Zzu>S7b%^rF{UpYmg^GA2jUj>ane)*II1 z@|mtOK4t@|ckn=UWBa7 z=&zj--fyAx2PHy^nMu}n&Ko0l+X!N`DNqq7*%_e?X6u8$xVkh%`b0>9WDlQ>6Wq{} z)~Vbe7$<$$eDYWh>y>4);t?0A<_EwQ!H}V+P$(RlOPRKP2Fx#`CWwK_9C>e$B zgm9rsnq_sbZi4_4Bu?W?@&(a267FT+8w+TjQDdT7RwBVrM_z8n1nKQ43SiYxXr<#z z^d!{Qg*cLG5eY1FTjZi^+j$lgWm8S?wTz2sT+rn9o))Qs0td9@*s<*lr8Zx{6354J zSF}E6F4}UYzvz?tZ*kYOEu)?~*;g^w1FYVJIG|N2H?-slSOEFb^R8`NmkJw85qCvF z6ait$9G1D0-O_3#-nTc>!!A4;Zx7;5$?X%CLNxN2neYvUkboz>V3C(|qm?=eZE%;i zCWLdDdK2C`$T%tCSLA!}p?i?fyqab5UXk)%I;$5-e{sjPG}K6)wbTIvSBsbWD3hT+ z;tnE!JJWalo@JT-Z-`AtJ%c}1Ot6BM;P^BZJo%E)ovAji!rmLdo*nCr8?=XX}Y8eXy-lM&IKz?Ju2%HcyGV^$AS(NA@_$-%e;$cse-MTwa2VWi7d5_*MRqr zm5C70e}|0YQAEt`?3`RP+cU>sbTbGn8n&j`DJv<@RLx6oy7_lB&=I_F%(i;`@Mmw` za*1%`EmL)VsTch02jsg4q8(OUZr95tdCwAg#|lJ%S08gK6qn*P|oUPwpK&2ewc2AJAk8m0*h%?ozNWfiv%CXJygQNmEA(bd;{`DB9?-?@&i*kZPE5v#Wmn@%$Ti-M&x|E|X2+8KGGxi;nw-%b50H#X zc4*HV%xK3S7e+0#nME|*55GO+5^Q)?$-I<#LlZ_{iAL4M^9>br#byrdtWFLJc3<|# zNM{9csrb|V`1#(C_qNPK>xLj_jz3(NSGapa8bxyL_r}QNa>L7#*)V5!Gax=CjjXTd z4?`2cfr4?(?4#?o=_vr9Y2Wie7Djt$QCkqc$*~mz-0hjk!LyEo(L0bpq$Xh>6UI_A z`7e*$jCveP0L0a#XyCJb(NyN63zm_;Rjk+&Ef?CXkk}ZWL<+@Ile=QPbupU2STP0M zbgw>Y*%+!_?WJG0fi#N~!d671oZCiUV#Q$5h723)R@39+-OG4AYD5?#*|8LwbMIP2 zG(>{i=U@P(aABj)T2F4-Hr_Z$OyIR?6>Py`#zCXK*$9xsC4HM?EOO$AYG37#zE)Ov zHqeVEu2GObu*AAK638Ie+a}`|Pxi5U;jm|Rz&5yJZzKWVvh6@c=iPx-t)x(Ep4;}l zaL)}kknd>5$6%4aLR5Ay#Tj`gOZ}f$kU=sMFZFmSnR+Knc>j|e5-$IMBQpLy4t_Pv z@`6emBj-`i#l=#(!{>@z3L*=$yL(&#GfAADbuZN3@R1NZ=4TFE7L5~pu&Fv9_&}=9 z?m#$TKZOVqmlgue{AkHm5rJ0Cv0=pus>3Bw?XT)Fl1Jg#6R*dG-`g?9yoh7C0ao*A zi!|{P@>nHx{K`QPdIO10pGLFY<4kuYH=Hm^9UGc!s`KAkxPS}vKO^*5UZ^>_Ac~7s z#(Wsotml0$8myj?)|38L@jIF^%x9#+^?Z}d9r-h!T40k`EfTogzN1^VJvwXCrD%vp zm*S|d|Jk5HPKJY+h9me}WcgxZGPT+w@Ho--G4-ZHLr{&K_C-d~oig6m--gi9C@X58 zu01%lv&>}9?iit?KptqQhLRg`Nm!!@Lc68N0h`|o%FEb|PGEs>w>2}_QZ1lfB6H-m zG-tNkgxT-ERVN!o!IXtbnBECbhW=l5mzLpveD$cqGzzA2i6Oe3n=nFxSwl-}8@5-H zyee|3Oy8&Wor$BI+*LHT>7kgl>@EnTkeamRD2Nx$8g0hG1cj%JZ&`M4L=ve-u#l-UCV~o2oHuYFc)v8qK$AFcxxe!Q)AMR{zhho@-+v}I^c*(e8GX8*iTnkwV zgZH+|*Wo*=S8r(LC$y5xmnt|uPT(Y9Tvs-MtDOc2IQpxb7;Y(+D zcHJ?YW3uB+gf1ml46m&!Y01v8AL25zZBwF|&_E)U(Tv@3l>6goAd>QtIO$M_B|AF$dO>x2TL7(5YBQpMj2*mJUil8_ zsa%rRacv|$h&VAf=oNEsV8p#_us#ZZ?<+^zteY`_p->+d!7y#c~j&>13C?@B*L^isuOJ3 zczJ)%(-E5ik2g2Ls=j&?h~5RlQzlA%xr?xzb^_2&$t#W}`2aSwXW1;csOvk{2QcIX z()Y?5`V}j&Xc!cIkmZ^vzNxa33MMO+ZG-6aDGymF}1TS(jDi9N7E zwZR6c)ET$wx#PU9Xde5>5>sp?&#D#gEsga_`{CEEy(u~t zZaUl3la%F54Ak?Ig1oNr6gTAHy*GWle)?9;$F`$bgJXCa?pkK4BJ5*X(BQ`tx!Gtz z*1=m6D#D~IN_BWBVIcj2g#m1FKaio^@-Wy}n!C{WIZng03GLWszuA``ZYC3oL+`NI zwyLFsPTsB*ClF(EvsAOt1sW*%l8A8NVsmzlY+h@h>s&zi>Tj@yI~9D-E^@h`jSLIdx6Uc;_)6n@=^`$|mnvamU&HxyksN-ScuiGs(*5AV8k~ z68q)?aTh*z`}ZXj?2T(1 zWS(tVG|Z1i$nq=|{nm^g3x>I~hMU?_cJ>3txE`Qx#T~baoTR9P=s&b5(TtWo)h5%IT;0aC z;o5nvdA0ud;uSH!MrtnIcETY;VLw<|SByWfr_Cy1wJRm%>h_SFky;CRg|+6@bE}^O0gp6cw#}45*bj{j z!H-4ROWQWlAuZ!^?4>y^N>I1rk<-0mNul?HI21cCmo}lN=?CY^Jrq{5T@5+Sg?tQr z?fSBi^OW74tCT)HgDJ#h>o(U(6*+o(G$+@5!!ufBSP;xrMYGR^P2#d}6M>+%Cr9K& z7BHiP3X(BfSjX)IUhYp_M5u2}NR(R2dEVjnMi#Cof-Jy?^!M;^x)M=j4y(3)~l;=j2A# z=Hh_D<5K=iLlHN(GbF--+%aEXRiA8q@J#U1;D*U8J&Ukb}iZXZjT%S^7idoYjKh+kqb@{G>( z4j-=8lGyA*n50XJmk*)&B-8@_Sbdw0?1)I6S zUKK6BHkYYa*EHn6)&HxmbI}VJ?#H`tZlG~5_=^u#bo4|5C_%_!J7fAo+R^3T>_J zPAc>;avswv$lfFTO~1&V@zW6j)IkNg z&o9!)Okp(-BdaEBnqO4q`XyITgfU(~*?H5;!N5+z#v4Jn-sm8r)-YcD>EepeO7C82 z+B6<08PsdM#vJF-D>UyNus#dnF;@4m;qwhY)_~=mmV5ZI!Z6LuM*FeC{;N+Aihm%d zfgAmg;=AI4mz1Zy1;{`C#3OcVNuBTlg&KKWN;R)lUJ+3%i5w=b_-721xk##WR_Ay_ihG&k-@{uWR6_}@g! zQi@CczZK1J``$f-nXkG1@w02ZW+z%7!#ig$snkHU7Zqk@WkH3~=hpO>92+O+F*zxd zJm}wh@En+|627>C^Sr_1c;j+lLQ{HRCm>*TyhY1A@6N}c{}ud=yx|H$bB6lS!^2#e zwy+?%U1_D0=YOGTT0BP{yfl(jF_HHOz}T52viAm0E%NNcVUFQ-khv0r#> ztW9r2!W4~o{!5VVy-VaU(*yq}jcz-#{rzB;XmC?H^}dJE z`G#qR!A9_{ubVMSXe%S8$oN8QI0ON9XA2<;%SPS&S*t(R`&9 zL1X^bP9p8sMn12*cMW&1)o-uw4u&OSr?if|f?+?1^SiaK;OBMq_UDoVRT&)H46bIc zstgC~ABgfJXpVis7&7$1)f9(<(aO6^k4;7FZ1$!kh5Rd&3XLNo_GvWUW0R^Z)2AYZ z-v-MRR_G76#}-W+nqwPylFEB=Pm46rEIKO90TFSiEM_y&p?(K~>%gb0|Hp!truMU` z5bf#OVONQ$osyc2Xp&*R;@QOnkx5B2+uD+q_l$+5$tO>qpzZO=Z-4t+0u91amhep* z2*Ia*{q*Ua{J)F0p^$!J+T|*&ScNl!1MvZ?GxG1(-@L2qq?*450h>zn8b{oA&dEzJ z{LZ)a(Q-Z5VF}75FPC0vTdpi+umwl>YHb^kYaGk?KqYxFr=VTf*1W+mOZblCB(Q`t zlJQ$cmI_*rhjU464dsF1-+M^dc5HPmaq-NN5)=~9`^%;GbLuK5uhWDXvX}kNe2&l$%uJ02SgjS`TK^b}* zY&|DG2m*DL&!FycOd)e4LG8}RJAJ>Fa!jq;A4CT{H+y~kjH75_yK2QwRnQGP)6~`$ zIBr%8RD2~^f%cfVrct!jvkkWJ&nQD{9;0u)B6PKa?jm4K-^7u=VcmNF#io9OH z3Amo&f|ms?wkW*Vv(v;1`x8X&Fw6?OHA41PSllP08`=S$U61H5*`iPQDu{TymrIRl zY6lH-kYTrxpP+w)D#C#0MndHF>bW>IR?f8sS5`F3wlnrolV^U%Vwi;;n9^$cA+Yic zkvcC}iFJF9a2-NIM#d358VMR27gQxyJ?I0>?pU!ku%qCPZ*aJ8|IINSIHN(5K>Ny$ z9-JfKj&4=7M~}0|=)@V>i*ppjTV@DVs2k^4hyZYGKhAwSa_-TS^O3r8K2TrI1DfOQ z*PHXvx^q5Kf6m9~(0NFY&I1l?pFW+B*QxWddUZZVx6ViG*ZC+NJ0GQI=K)*x8%9X%hTr)PXu&oLlP z|IY{L>$%6h82!Rw-96*t#&LA&ycP-fvYHVz9hoZCMyu>mO+7U>60*jMhH4*oY$LC` zwB^Rf@1f?}Ma9lYD#dYW%saVEvG`M!$E)~W2)|~gyN7Y2PBr?9<5=oU?Gy+meQDNeq@F0 zGWv``wISbYJ%+-G>(Rio_n>*)8TS#iV)CkX6;ejG-Iib=Md#lFR{AJ+sy?jG#Tw_C zujdbAJi9T4Qn1R&(iGY^4JkV&9tqv1dsd=uQrfWdKKyyb3#GvT-S*d`cpy5+-_m@f zOUb?G_i-R}A*<46$Ku~3whJntm(qIQ9@scNLN#fovO?%VGxp_-Fq*8j2sQ&DClaT$ zz-cARmwK_zSJvh`BX*5=$m#`57{7~)u~H_;C>A2fn8J$ywl+9(*X+-*{#ZF~w?lQt zm7eY`+p67F#PSon);e`uUk?-NQ3>xGWA+pJYC#IWVy3t5!_KQ{g^r@xG8kTdr?~B{9<)P*Fe;vZrSx54!nSC=EL3ErDM_5|;VFV>=Pi53MffQh`Um-XEk@h4iN-(Pv! znc8d3D(g6oXtv=WXemv~Ld|^Z=AiLTA(KB0gysh(Ye#oaBcml)%D3C*z7&xm@5Qi->zIA03=zt|AI$v%VW>o&%Xp2{{7=wr38Y{^Ll-R?o&jgJ>g? zR7Jr=nQcF@WbXDaK7^zm%>#9v4bej8WU$fluL(vUjaf5Y9*Gp8?& z2C4O{r^9)T&5_fxWQBN6{{JUG{PW*ur@ucr`Ohb_|NUS8eDcE_{_(5R-=F^7{*Qk? zJv}-3&p*HYHmoJ{K3t2;f-uWu%Q6 zU+OV0n?I(sj7fR?>@3SaMXVZ@aF2vK1*(0)fogct4buR`pN*MoM3frFpG5+VjZy;_ zcQooxQEJ0uic*`R)TSu4F)*hnwJAz%ic*`R)MCaIr8Y&WJwTKiDs3IHYLSaG{kiHu zwSS~IHKh?OVHZgvD^cFaTNV+3(y}G&K!6rgZQDYOEdi_#yWk!-gpF1y*J}J8IA&k< zn>&h>rS1=rC}yZhsaHEE7o=bb&$%g+K=F6PI#c*KmfrGGv22M_5fydWS zks{s4aJ2)Y7MZ{u7HwiltHPzK-_eYxyxh8oFuYz`_ZY0qUJr|ysy=f<^SqEWSu^F$ z{A|Q&n-cb}xNTOgxIm=QS%Gyl)Qs**f?LN;7fWo(H8Pi6Vd*CVcByt1-qQ3GE=fqy$KCAxAe0fkxPTP`^s zrLV0R)-@hgL~NP6FC@)A^1)$Ieh|>#D!JH}ajiPcv(3_c|JVGu(>}0w-d}27 z9@z6zjI6RMWECt*&{zvRZ8c=;QGklp`bXOk3C!EUlL9z{NAze7F1mVC*b|9q`Y-qB z;1`7WKy8Byw&JQR#_R~b&_qsCDoGY(Q)MO3Gq*dzMZCAb443r1Bh)j!Q|v5*EnRcU z!^HQ*&4Gy$`V2q>J$;pjo{OUhK=E$XPN$MbLNJ z?4k>g6L~MNRU0EKe#b-$D@laO=npixWg;ctD!V!9R?v-Ue47{Yj%(y(X$$s1q%4}I zu`aVMosMGFp*vNn5e*~t!$b5(q3OTicL|G;MK#Pr7jYMC-L$g|dz{{_Fa>;#-xn)` zW}2*o^)cI-q=hJ^(EXz~G`3f#_kkce9ptmcBCeCoaj{67H0(W1r6PZTQe`h02<<+v z?cJAl3*X&Vy`2q4*+;s`Gp6RZ{{pr)bEnRWMSA}J)y2!VujZTdkS3EOn`j+}JJ>av zJBN9l-iocP$UVh6i>CA44bPvGS3*_LVuzNk&kV|GKuY5J;E`8mbHftlC+LQEXlvPG z5!SQOgu0gZB$ebILJsbPC*RQ32y}$U zozVgACjMk9mBm@ON8PqxuI2r#l(R~)84t$3MJ@Q2Z6OdaddcR6hVEBNF!I9hTG)91 zhnL@~A-j<$R-dynIZs%=J~zUMu(oZ_vBj zU$6iAC)=jsKI=)r%7GM+cB^&s3h3vaqmfqSnu(GpXxUZ60f#>6B9}|lDpHn0@0)_B zOc6()Y9tJT6g>MByBq7iiE-eLbKQPwr|ejFY?PyQ6rCG1#O>!?9HlQFgTmM#knqb6 zfu!k%59CLG79?_5B?Gyt_pg?z?yHfhgl6pWn<%Z!Jlj;vuFa2Lf93%-q_0Q#pB?fL zA=7?G!PFnTsS$Ztk8O(&W!fuRvU?iOv(58%`(Sl6O$(-!rJx2&f=j%(x+H%@P>1&^ zFJy`0=S#VvJQ%3Z?xEXw(JsnourZ6da&U+Nw3LNc;idA9D>>Ih_wh@>{AKX~V zgY{@9Ip~X*gOti%3A)I_p4KKpyI08TjIIcS|8bsjrTG7hFzN8PE*Exh4qgo6)ljy7hpXtFmw;y%<>#BRKZ{-7)O`CqmwdVdW>75Zh&LX*t&MH{)Ic+LFt>%0fYC-ZmfXf57h_ z;)e2aY~porKQX5}yDO^lRYx|L#cI%?w6f-qwWUG*hs1p&7TTRlXvs;J^r zCaEem@Z`yFzW#bnUh)DO3~|>%yLMU7f`KrKSx%sn4wD5V#0RQUb!1Zw`ypFZs1D7Z z+A~`!HajXVz;MvJ(2;?D8O%Vpx#a`<8I^ZFG8ely+k7~WJPg;*F@MnP@ew)Wel~az zW=y8|ZVve1arNbg4`hCa>0dJ|{9(G1?AwbZ!qGM|qdPKFJ3gr+Dl7Y;aJf@gtFeiC8&o}~NM<()Mc5wt)X9yUTQyfj`;&KayvxXxw4<0O; zATHB%WPWL#IU1I3#1ATg^9Hx}U9}ZG5EtPSnYF`VhX;hm0ou>AZO9&(n*i8nrd8F%F0ah-q>W z(;)+>ue*O2SC=Fy7-SC9Okq>#Ry`ZD-9e~ETC_49Elak^2m0VnPP{MPiT5B5fpxW^ zVy15}xbZx$8r*?46PXipSCO@XK894KgCLH1@1p;@g86*Xw#t_G*+Gje_IdVMCHe8I?u7lka0 zH^*N_5GF*PXg!e?s2UX}KH-{`21Nkr*brf4PNg->&7jZ!4mfvQwp{)rON@6a*Sz+-V2WAu z3ixj{CC9sK_|v{}qp5O$8Jsqmpu9v{AC<#JHgW^~PHN;0`3C*NPZIqjG#vd3#d^L#reKM3;u z#hV5^u1p|U>Uq#X2)HWbhLvkpb+OIg{Tq2UfA(blM8kY5)4C-t{-RVN! zvH32U3n_i~Y%#DyFTt9n8ey`oJP`<5^=$6hg7j5s3Uey0>)=szBVp&1{&aJ5MRHou z4J%oJZh+7fhNl5jVKil-B)i63-k9!}<}JL)#<)(?6j=~Vv|uWiLWPR$#3`G`nS0HW zs^H}oiQx#B9>{^%>Q%#;SwJ&GR&goAo~hQV$hShVQZv#;M#XKIjYsN*`hldpDX36d za9IS9*j_HV${1CQZ0N0)bOva*m-JmJhTc4^vY{nzsGY%; zJqh$Dfdyy)=%SGK3eZrJOw!c9oYqBz+C9y+utbN0LNHw)B~#$Rnjd4_A%IlX0{@o+ z1q8O~ddMANO&ck@W0}VJ`9}W8vy9GVu{vjB_U$zqI=*L%a}DwQPk6u1d(ZJ9(CwqS z5?TO8kL?&&xka`AnAw}9;mi!6hq_ke0$i#rt;>Z7sRUmxA8*sUR}}ml6IC#vRP+6# zut+{5@8q>!*n=GHYt1GOIaC0!dz#V29Y^>y(IAvEfnhuPuE~P0hO*`kJKM&>9{*5Z zGBM;hyw*nr>wb`<|BvhII`wC-v;U9l>zS85R3ARgvi;BgkV9+E=jW7;TtW9Zr5`zz4uu z`;y7aND(NzGp=8xt&MiU$(o8)I9P>}LS~s(1*$xyWo>jU3%**hB4k(?cFhbN(S~aF zB@iKcNL;UEzo8%brrHouZ5FK13s-zoWhE6%R$4W3fO{5jL4IU~H2bz;)E4gd1dvoR zC5}rI69i^WE)u!vN6Y%N-TO8ihqB>9uj@H^64=uI(3G66*bOhU9@|A@J8v*G!`FrM z>!j50ch7&|W|o;6trO|Pym@ooR*M2s*|5@)up&@ptuYPLBC(!_ewQyHQOIRm4i)l^ zR)dy&^_B)JEACDY-|<@YyYkf5MLAqBw4}v~g*~m08}N8jM>=BLxd#n&jH|5wd3!S- zrB;4io%|6y9bX^6qk&~JSgA&BtaKl-oPBEH$JfDss5r9d(zN^5(T{1t*OCh%sv%55 zJ|7~{4E}N?0}7kXNo*f#hy;Rim=R&54oALCm_S%5^xXvUwe@L9!w_23UCFT2m04L%MZH_LSs> zn*fHvai-?vH6*}->#(FegR&jA7-zgGi!Z>tf9z#i({7K86~j6 zVQh$DlCd47hL~fRF-9@PeHh|~el&*{NeJE=FS2K3s>_~%*H;tOx^hyO)7`wee)`pr z!#s#48!kpJ0Nuq8xPWH|EdW^)N)Io_TLfQ7Q5G_5RcJ(61Sz12>jl<+99qsss?toS zXc-pINQ1&3tJcFITagC_OS2Xgt8=&61Ci(FEY6}-%t%$v#YY;i>j1`!YHY8 z#aA<0Bx_!>q^t^dPIEp>q`1>kv=nN-Nq?zwmdvO~XE^gFx9Jx%sFnKOGl-4txMfY@ zLY6f1B4*tCB$uhebZ-=YT*uW77kaO@rbBR7Dgl}&Y>pIfMEj>K&*WC4@h-B4V_`yp zE_y1wM?=_=rjn}A0tX-k>%fW76no{@HqpZA_(^q`{o{SIU|0`R4V6!I8BX;i&&Wl_ zigE+VK?RnXr9!IrRmKVjxr-hQ$*37s7_ANqxZ#nxguAcV|62RN-XvJBNL|RchsW3*?Fyv#q$j-X-Z2vU%6-ckJEO2b4w)m)bxj>THmd5DNz0A z;Mvj)k}t`JuCILPS1gsqm~2a)K070e3P1Z{jucx6sv@!AJv{1n7kF^|?BeA;v$<~YZ#S8} zKV1e4Urm1DwLz>;f5G(KE*QF~_nbvIp>UU;c0PuC*!d^@`7Y+XUrJ(2{mQs1K?l2K z+m0jb`T8yh{NRSRyFBMq%d-Q_{C!Q9tad=fmQalHwb?$ENLi6V^L%8NT};-P4Zuo< zr<=1vUpkh~i7AGYv64C9u6(F_tY`AkSLD_ODO0#ctrNNY=59jogUZpcZw zX1pL7U9e1%4J{K>lMU1=S)4wkgl2(8$Y=IZTV2wd+8zt;mxTiNQ>|%{BGKuRlXj-8 zaha^C#yM<24Gq2#%$&TU$=d#ky*j$)Oo^&C8(P$+KK-`2;Ugr`AptK!N9bDRXicG) z@wGLGU3~%fZI~7IR&I=8g$FFY6AeGbtl<;=wHO+czOSL?|?`>76Zvqd4?+%31Q5%MVK=JE!0IR1x~4LEp~5b)Z-G zbIlX_=_H)K#l#Ca0AYx|T*E_Fncmdat!!J{x4fJuc>EzbchtQDOQX`sqJCdG_S1ub%xAdGY|?9vqb_Y4J}_p3q#%;*Rz{ zr%Jk92H-rw;r{!Dj9)sJPC@VI=+>zeD@=JB@bJ_)F~c%%vgrKn?#cYO&*o2Tuivg` zW=WBro6fpitmcUG8+39Gjnvaw6|%~olkSHQo}D%=*$Uba;s2hIj?IR@;O4ZX z3oVoVx%G!P+$gMhRuwdBP8^xVS{6{1yzx=|6Bny0qlN$0F9_BUeO?!p)ZEkNt!NlC zLQ^y-((J0x%({3XvuY#U7k2CnRwj_kU6JXpN(!Itm}kvj@i@L~{I;X=b$qcFd8}O0 zV*2gRw*HLYeTnZjzUxaT!4IQpA@Et`oqaL~&fvY*`mXEyyYM`J!S7H0e{(4R_;3G4 zj^DFHiiBq@JwE-fdBXr}KV={2-{<(xxj??2uerZRea!h(peU)aEUS`k@+Qz#6P$C{ zwGQfgFDS!`Qd&KM^>b=nB`7=+S$IuNc6Xp5V&Xim{GalIODLU1o;D`uTUhb6JA6PT zYqoL8Nit{R;_C9dudW+Ew*>u$gpTzH=#X=`+v4UC?}mKa3|G)IGM75_>K~ZPtCfD3 zEt+t+s$FqiUN^af4muzN&k4rdDBD61A`A-99_32#A02GhtT>}3+p*gLT@toY&6(@; zR51M&QV94WKOUX2zg* z4-x32A{H~XpzeP=D?A50H@3Qs0?%?*nRpl;#Pp&Iti#`gykfZL-gMFd%^!bZE)*Nf z6stJZVK2|EtD?U?7!SR$`2+Uz*0Nanzo`q7L={BNPUw6&S{y`^pTS<-0~KAczd6+- zX3vwgmmO#Hxc0(y0{O6KoJgfPBy|*VNyqV(xXvtEJQu%2qZ&SGohvanmM2jWGl?6; z8-zd7U>{^4wrSl5v8HZ8rT_4=6

    • C`asR3w;Xp3T{Socb?<8t&gd3 zb7vW5hFG*eNpOJ%@g($~CB%jY4rK-A)aGs@a30ndhWPW_kovjKk|#s3tne+H-gEFS{@_sbEx{ zajhznpf3@np`nXU77;y&9E23)-@M?+v^d!|T2rn00mr^bR@T-<9CO&~!9Pu6DoB&3q$g6#ET^DaUSxMWCiAL8U|D zMq)3V=0>3X?*U}ebKZkHwgeoSe1CYk-^wZ&-^#@B7f%YYd{qi#tsWGchyRteE3yBX zsNnn8_!dwl^`D|Tg7Lo>M)Or1y?lSN!q_w7Cq;04#~8d5E{xQAS8xco42@ic?ob8j znkA-kcVN|a@YG`?UjF3Ta%SVv+!$p0BX>*ZJC@_4zf{EjlO6U|K)ykQAv?i;-rku{ zM-sLrl)sGAZXq*nh>Lq9?tM2|@PE4tm%OKF^3hO=WL{d6y%H>7B>K+d;BU>=wfERY z#3Cj7l)QN``xG@h04+x{8;iu7o)h(I!VsnM+;7zne|kw`>Y`Fd*C^?ecF=uZHY5@L z#CRn&tTVPEU?ElLL7I;$Vrz5^U!H-kmOvKD>rI*G7Z&JEOJSz5?&QDCR*{{4*}LDV z2^mZe9^JP*X|=D(S+t-qiwRO{Il{aEVneVr>p}>`B0gfh)Y1yD(y~BfuvSJ{g%%ei zxkk=83seLWxFF0IjL6d+%!A{sqYaT3Z6mYo2*F0GaLC!MfW*Ey+5TC^qn&VY75%7> zD=8#}WGJ}UkS2@L!?h@3_GO=PyLt*EEgF=9MKhT}f(2t5c}m3#&m5PVHU?9JrMZYa z-QQz~^1-`FnfZ&2itY5!wF%EAMuG~1xEI27nF0iJH_->j-{e9!W>y;VouKd_&Ev1} z<~2I8)6ffE;p-t#?!JRRahAJA%F5(21|=s?NUWp#-#v!yy_jas=L2OQSpU5(-*C`L zLQtQu_d9$(Ez=!}T*Infhk#m!C3%c};*DB=2CqtEkuD1AEe60A!TF{EKLJcoP=Ol? zOYNdvz)m=xS-p2$Gu_lR`@*sLhiAVj!sD~A!yUHE?#QZh!tSK&%IP8%Y3tz@lb0nk zmq+-g&g?3nT6-E3msS{DJv`mi#$5h~rm7b`nQ)W);Ytt>0&i6Bz4zd81cQr6I(O`T z27|69&WD7RqE_$g!5`o^+evVPoj50cH4MOop&16iM9)N4If`+5aNRIRg_JtC&qwNF zghrH2x3KwDFYWr-pa7>?ma)R+PY4Yc`}9Y|@17{$U#NpnrKaW7A*;6pp@)ABZpj81 zU%7wSZ{-pwNO!-g?F8vu4hC;5mIeA!nW+$Q5tATN7ubuYgMfz%+!IZ(iT$-9sZGy< z^%(Rg9Oo$zqfoMSOFl^e*l`4*7hK-zo@;{?M3Bf3Zj72Fxa7BrC_lt)mZ3E&=0XYN zILFUuwg^gRCA;o3b^uQAkM6&JEG04OW1;0ZwgiA1-!tB^k%xz5nt(MDAxtaZSHjT; zWga)N#HoE6S-K&wE)7KnB=-wgJ-w|hUO$9}E*$Mi)r@PweZ$&VWuqS#aS5B49zcS% z-brZdki-WX@bw*2|3e_qhaS*S#iN1p}9%8jBp&GOsOzbGvknS z=>Wk>e$z44kD*dQvr!Z}0G0+3F`|JR8UT`A7;W1r4-E^s25Mj)xSC1bg3;4fEQCr5 zZ#aM~TpcJ3E;{mu>q*3o(50+66K-225H#F&EPj^-cp6_l0eA5I-`fbKxXM?X(r(%p zYr;}fb*V5YpRcpGRPIi2V@L7FSS==XZVm6p#CnWe{TW)~Xsia`}O zR<$5SQG_j=1ivSgxRzjf!|mN_N?o4> zp*}8&wt-9TNn6bbyI*AM!Qpc8l%aS3p=wb-G?(z- zn5_bTNYW#Ezj9JO76P?xX4!1Wi)sD`+GYFv3F^8s1!6?vJ2j;IVC+tNZ_DL;{-$G4 z7D~L-`#xZ{0T7~C2B0~dI<(rqQ^j7U^x}oLfMlPp0l}RsRKuSysO`spOMm#?S=2-$uGR6Aj@I2|exh%$gq;$B z^X<1;1NDY2IwDx9OKkixomf@SIg)G(80&X_K~{;lm{TICn6p%?qim%3S?F=3GS>0b zSmzBRm2bHC{}=JKbmlQqxA=cE()mV7Z)5$t>I(bc!EWT=4bS|{!^Q6beoeb*-_uVR z`*S0057msr#ooi;Z62c>Pj!u>r1Afq{nyEVVvISDjx3_2KdCxT;TRc}(owYS80lpt zMX0+pFCoTjhqA&|<{Ha+e|wE!T6UED94^PC#_G!C&SD)w|8K7>|8|%m>7-Jons-dA zY|yFuC}d92<(kntL66rTq@Ip}Y9>QNWCSAO+6DrzcgFG~d)sNx8%fO<8ON)`HxE>c zgifCe`9?5Z&4(^U6}yRELyOqdbUmd|-CTh$_?1r$xqWi40VKM6VN84_s3Bw4p4y9``zyKaSh906Q z?iceI*13J=Q(NuWE#nTbDjn?Cbledv`l-u?mk5u{KH0dnkTfQ$)Y5pRO^f zy0RKmFr z#BlI3aakww7(&!3Nzk;bv! z&yUi=+>)DEemsBIkEzd_(*6CyLi}Fu*UQPpq3MU0&Dbi9&Gg(Y+)oKQzZftlpN~cJ zElb`$t)+tn`kf{#L%cWm+8gbeq?-tInREjWgOi_`yJ@^Z6(3|5SVMX=*xt;0r{aO7 zY%z`iJlz1_zdA!I4UH4#bQ@-TB!w$@a?3fIo6BKz#j-zGRnsd)|SNuFPE58 zVdmTp^Qt+5$|h~ZDCk(wR9j6tKhaH$2CeSc2uYd~KOAbUqQHr(f?DFspCXc*isuIf znO%3w!=A}Clv3tcWdT`>iKx0nTTOJ6H8&7J>eP8pfeR~QnAPU|IUI>?zy^}mrh{A@ z(^t1*UU!qbJKDf`&2k94_KWmaWZE!$>6@zE`?DOFhq<@;ukq%OB; zk%L>q*4J77HhJ_GU5}$g_=tb3V_)?Jl|C(5kn{FUtgJEoT1~-hJ72j)oFxeQDzNUD zWhxOT8o%3dXJoGNh$>v0juy&|e-#WTpaXb#0MoNC1oe=@N~P>-w%aTQ-!$^Id=a6{ zo@RBC@Hf_}s(W?nWzwA-H6u(9i1tdLTrd^1LZ>?ahr0ifZoFbnj zAMn0Z#1~O=IB*feS6O^X0jwe_x8ThS7Y@%=$PeKW%nrWUz zdDpTWzFqXTJei8+vJEW4!`^Pehku|(-iP@!io^GpKwO+iIx%nsOoik5es^6qV9Epg zZ)Ki!7E*~-TutAsGUU@tizv)QXGCG5RJZlGN%a}z$wGWm4~f30AG+P|MOtLg8i_k znKBpf=4F>n1^s?6gVW#0uLO~MSo(qfqot}vJ7=}_q<1#b1Nj9ajsbxK>QJxGJ$_O7 z7~s}=5mKLC#k+gsw!7)6TEF(M<45}AX??vOq1ra0nng28TasemWpb2LMJ|XE9l3rx zaK<;9!{@p4afT!je5hL$CJBd_d{ZNV{`Cmg+FqzAN?Cd3erIMw^bZAQQ7`q^u0@WS zL#h7N=}dfWs6PaF3--;{Ii;7sIpA9OG{&M*6wP~<&;ii=O=9G}wUdWG{HVG^J zVTSBE3J_7Rt)>6lOQK;A(oc+0^k~UscLjCQw0p)xWOZ z_ShI5b_E898?z$H+MXzB1PSU&2d8i-SiQ0^whAODbR<*Gm~JjdnHA97myz zB^1QTB$5N_ugN=Fi#UUz7nRANlMB%p7(TE~jnGh$TF6*xA!QvhO@FDCN`R$i%*d_) zaJiWwR)2SLtw>&N5+!;u&0`M|$vJ*??C<^-xdgTC)ge9p?wayJM<#@e8IJkubN~|; zQ(kh?)oRbrUM3;cj}B_~5Y@`=s4K-1v}3P@ql^{@5iHyg6bdxeJ7)0?3_2k_O4OV9 zI!ewRYSe}wE9ks#)iwhVRV#%RP{qJb5+~(yGjQpf7u19Yw7+9xachK<5%7HICr zOyzH&ra~0MsJNw=vc}W<)&r)l_0-KEbok>F=H33T~YPRP|wxL zrJy#tz=rVr3ny9A-`rx#wUe(uRUW}w}!F3aZg6e2UN)a5T+ zIyGc|6few(y$i1ARC00cLsxyhTy}j(+mWMo#24;?(+DX{n9iB~oIfd9+xf)y)>+|2 z08guU-vV_+x+qyqH3-(9B@-&?8K8+W~f91D!UtN zJC5BAFjkdQb`HDQ-exB2pQz}pfoehrb$bQ=wOhiJjEn_fdr>!PQChJ44r=x;P^8{8 zC%-#>9`H;&Oc_F5z@0O(>N_ub7SR}364^EfscK6imgH-Tt1^0_BUiVb+cXr|KhlH> z*al}aO_4obR)6{Mh^AJ_z}+Tln%LyJ3ecTMSxRYK=YqcfV#cg_wy`WvkrG7;4<`J@ zT!Ebpsn5>9w~o$=?GFsb$JoQ!0;aiP0{(LaSeN{Vy>b*pv2q zD72T|R1p<~%A$1wx!oS#>+)j?<$mp7@Pk5FCRfzE8mYQLv!VwOSqAC}R_ZGmI<|mq zh(4=qY(Up{7geIpA?AiW#FG_3#{ouY^Fja$@i%?EnoB*)0NH?oBQ)o+`LN?(x<0rg zkWJ(IIUEFX>6;#-kU1fNX-H9rj&zCOWVG?{F}1T1cPLxQVLI;4Nr-4&w{M{z7q|1` zftS^76|?ighM(PSwLH3h0zRsKLT(x3+cNEN(ZKAsx{76-nq5B;V;}YJF1P=N&*|^c zM}U8AZ}!`K$@rg>@BK06K0bSsE2gE^|1JD?^VEu|`M=IiJ%+KDljgRt+!>37Y;OFM zZzIfxe6ISDWD$2OwfkXIQR`X|!pl&~#OC~;5NEbPmr4$VXgLK*ek>WB>^-uRmONO0 z)r5q*`};CRF_bP>ZL_X4j_HPDy45B|N&AmMG$M^5GmB;2iJ}H&PC^-m9S?FXFeD_= zG6mKtacDjzM)(T^bgk}CDJ^`7Q3>9R+cXhS%S@6Z%7Hxxwbew(%yxZnB!C4jz{S$A z!xJf1)XkyXqi@Hr9y%`7PKQJ#f*J#@yXnL7Oong=ehb; zs}_QOD9T|e;-KD;dKvyxS+g^un*dl5L(_VL40_fG*lug|?iaUy7gnr~WYHkN>9D@( znX=ZvrWPJ_hSa+0cSPB+mj?xQ!`1ko(xl}paMg%DVBu`KX=vr8sR3S_VUO?H5)W{d zd#DtzJgoUi83iVBXP8}0#2Xi*@lpe0zqkabv|d%}4K5y$-*gJ$futI$N+dUm z*!S+T40N-Ak1b_NHhx{h#hP}3r7HIar|%PXmML@A#^w#W521b|fUwPCiuZR$1wFv+ zm^xWsmU-t3w`u8VEu6hn)0JGXmOFs7|lUI>fCt6vDR*t;$tL<@3m`}uJ((VK1^ z*e2l5{mVne(+CQAMrD#gonxXY1(l_N3LbJhc!hhvIYM~XBL42^C|AqwqBKG(!N$*s zi9|~F5;1Kizt}D=p+Psx&it$4NVA@3dMsm@E!Vnhw~7WcSpnQK<@Nx+_$hW=`?%OD zr{%aNK>!}I6$q)Yc3Zr{uceEWp=xs@f*yN2gTM@8yJ%khmpN*Zq& z1Zo8n$2cn%Ug0y8-3i_r`!HBDi`poU?E=Gic$|ur(idnJ1B)1drjcot*N2mFlcJLO zj^_aGXhj$xVzH6kvsdm5O;=6XxJFnsG_)(8xI>uw7HTCs$zAQQtY>wfbX@lA zpKJ&MYR|M`$em0UY72FGK4XQr@~N{4LhUMVy9wh9fyTh)Zi7bc5rf{}15(Y} zs$X5YV6}_z1%Gti7w8Q5Rip|GL?LO2<7`K)tDb963lem73t`v~NRf*U&SP2bvXOm5 zH7~j-#t2xuXn_vV7@tWzO>Eo_Y#Ol9Dtc6v)SB#Go#@h`-3E5{wnS`It4QrI%Dthk z21}?=RX5m<6(E^YGNpZfXsZl#8+7^|-rbPKd-IfOZ5P+x@xfYwtx@OO`wMX))yE+q z=1*cM*(wzx6u#$%+bg>Ly?|h?T7ynZGo3vN#+^@7;-Ii(XUz9NCUd$ zy_2;nxJj5`C~f8b)d+@Q0$8_{crX98f6IqJ`SujHQYCGh&*_V-yHxc>FOB1Y$;yty zHnARFfn;5*=NdIoVZ1EC2@+c+)g(H$AbrE)^QB_mup<=MN^dzYht96V-a?(YaT^M4 zz5XtA${d9@{NX6)7O!rwx%2iW%&;I+AGDU<35fdqET8PTzvN}T_I%7uqO|=R?I_|l zYB9-D4DeVRN~MUY0n@jf?2e1>cy)^C*=qZ|;-VPdyl0c$R9*QSo)1`V?(`YJiT9%` zM#j4U;#^J zZ;lXR%PdER!f85+e&rWOvk&KOJHB(*SAdxE+EOqN8-BLxXXXQej(3X=D4C4rIf z>fVpzhX>_vf<~$Aw*9M@CwUorMaz{++B5zyb}fIt&%MY~O>wO}c|=k#RA|ZGqtl~| zFR8qB8z^(Zq59J>@=by>!c2?!Tg)pM9uVgm>CLzY^*nj-Z}2v{h*aE(=~Oz2#qfMI zA**SzAOXMcLETywWyg2TZBkQTUSJVi7)V+Amow&OQ8UgTt}%3s%-wrvw|)fOxHf0?^Ar(XP2t6Me)5&H@cTh-sAgceffMOU%vP}Y<8wz_xQTsU+rx!GJc-D zRMy6}YV&%RrE|AjLyhGEa*j!w-<#HN?SGf=3yjRcsj)cbrPLc47C{v8DGBCj)e?fW zC5mY-w9Wg3Gc^%WBj#3MR8qA_-{)yV5qGR-yxKzANG9b3$*TE{VXEq6M07)lD%AJb&S#Hg({}K;`1t&E>RUh#>aUD$Vu6b!GM)tlI&f#4Yozk_iy7!IGN8$^7brh)I~5)eg#yQzQbl5iEkY1@a7 z2QP?tsX#I(L_*zgF^Dknh_QtmZsTzyTwlgpFD_JgIeh_$Z6WYi2rH`BIpQE>Shj!- zya_=Gvds)ox(jkEi8z1cVPs{mhMy&64+yZ(g=?MM@8T6~UA*m=e*>~N?6j}loisP_ zK21QoV3eOdj^@k*HxyC1f)hlh6F5N?6*H+$b3<>eoH~$=k6%9{)=!JwiGpf=gnO}# z6)b&JoF?yK$<_F^2#`$Z=kH)ko(A^(+C~rS0#^m5G8gqdATP1ENWDldK7G8QQ z?UXC7U-IeS`1u`fiby`iZWeSRch&giuwQ@*8+RQeH>rqv1+=U$9Es1cZRi8nV}IQxW!-k zDQyC6kt4cQ_mIrss&urZw#vyacF@(UnsI^J>V(0`sV_Xh&PGJtM%IM)z1666+PFQd zAm@J*gSxsfEhR}%Nm$99E1q;KhTsKwvI2Sc^Hfg3FLSG~#(z@d!R`EFja|d@Z1Y7K z;bN%3FYkcvZG;8##nP}VLi^-ixytv>dUEeqF~t|7$xnmoAkVdg+ph9 zG*&8*Zkd@leZg@QL{C~Q1AdvxpA`Z*Qs=VI_Ee5LYfNmLTL!(jnp{xOIKXc8f6N~2 ziN0{*T*W5raBl!O3OjNG^fPtNLp>wZ%#{A4Rh_ULcrTdD5QSy+7@54BQ{}BIsvtFI zNi#4xw>{SBkV61QqV>`ymxrLGKAC+S%5zC-I^9MtkRv$JMH~afd(7=ufi@y%_Qgl5 zHuwmX*+bLP)$gfln^mT;)FsuErCCc|$yd*NILR+M8HEp8a?*OPp=b9N{IRf=pH0+N zzh>jLy0yRftu4I`jZM7_bAs7Dy(*M|=J}pG;fduptln*Fq}k(`{>=MmK<j_=zKC2DBP~|J?2#5c*pEa!3b1**WkU&ZClCEG zdft7D>gh%%xP3j=)fTUZxH$)ifD&_&u~2uhl*jy%ZPy#AA!tLOrgyX9db_KRNgdT# zt#N0xA;Nm(0vC~1w~BGH$wARHojC!KvVP^n*ewM-$$?O!;K9990MBwUi>aJ}M0^UH zPtWob^J_a!i!o+xr{AO;f5JM*gT6CKX-!{D3fXc_ zOYJzBw2-!bx$t76mZm_hmVX6SpFvVCXO?e0E z(lZXSK`PMi(rrGeyyWXa@-R2mkCph&CabLzY-Y_}A184(-1qI*OIx5LXX*_A4c;^mz){!my5odrUldf;Atjsv7jT(R}LF~}If&cz@D77CJ>%+1QZ`Z7f*i>v}e zj%l!ph>viv%HKRIOKx&kMwK)+y4q4UY;4kc>i8VyK zY+PHLg!N>>8nfQsz?LiB6(QcY(WQ*yZB@lt~#zdCtI^XZpnZ^QZK z1*)FQ=)3L&Mr!CS&O$^J44(!iUYi}IZ(T(otYfP2(-XMikV-II!{-$bnA z)DhK~yZLb+Gd~X9iVU#s9f^Ui%{`ZRuY|=aJ0Zvbiwe$W(L@ z15|rrQms_GWk}Hp@qX2n52zrQvl3tN4GbnRYM`)?y6E?%k_KV=xoz<>Cu2Xb3MSvt zAhI9icoMn6=B>3b>yw5aX-a(1RGOwP_Co@R&AiLZKvKf0U7#BUB}svv-5&iX&svcC zitX?9-v0B4c|Tpz*v#c=eSXnnODpbS)Pz5gRR=#fO8GIEJU}+<Q6<0*Zluf7P>!nWLC}Z?mz{|S17EN7YoR#vrB({Pa+Ph9D!JG*QKHv+{adZGIe%SA%|Tw~t;e zvSw}b@tLpOECTnD6+|P;x@AVHbI`<}nQt7yv-CeW0=XV~^J6g5@jD^S;RE{vkZ#ok zQZstV3orFm{Y447o(Als&U+gs0TP7Cjs5t4kpwLl zcvW{x&^G0SIidcvn?!-b2X+MGTUDrTlo)C{ z?udywLk8~T?ja{>$tl%5lsYR*#YQq$?2=Jz?8S$qWopO$`fC$;@*>sQ?Y}%9-|iM=jHr)NLKTu2pHq_^=pT*(s(VJIEAe#YY`P+Q|tUA8l2YbH69`aC}#r;I5{vX-?zxIoPKg;tF0%X9Yg3apF_uQ;(>Zw zU#(QLSct@cU@ngvAe0;(VOV*@fSohTvyb#KJLRO~PK~{3VJ}n+-d`;g)(9qp>A29U z&~W3{tvGzLGKV}Ubra`DsQ=O`f_DXFfi#$Q;BwJGB2qT0vnfpGb6uLi=+Z+9Ztejl zL?xfZ?=N);w42b+NSne(K=}LF8R`Euvb#Io%E+PDji9o%l7}rI!;LYl3u3G7k(K@w!C@V^zA@efG(iJB;lD zTTkOAhRkSqw|o4${MONi+BK7V%@B5u%{z6cm?p?Mf2`lzeCOD$y101D%c8upGUEA5 z<-Yq>C_gSfKJMqs=7ha{4V_=f$hDuz(Am_-!_|qn)2%Z}-2Q%T3&7Gf8nT&)oj8ee z5ohxKZumANf+J<)0!hIaiu&$=z;$wm*MSJdvw=Q(Q^2~2RivWJ*He4)mT%FA96p0o z^=?MzCb=36-}BrYj3yFO7PpBTxvry)5ep%3-`$Rf!7wijAX50CWI{J=!Raw$Zd75* z??!4nu*CPsweyEJml?T+4kI=B`D_W@-{y> z^d=by&HmhEmyT(tZ!xo1|HdnCNXv(7c1zY;=x_ZK`8=JpW7F$L_gpO@dfr&(=@{sp zGG2~}=(CbCL(6L+_%s9qQcOugwP@q`IgYt;_~L!{dyx4QiIop{2iog&uzCrijyGcl zwR$3&RE5YETRq&cXz|oSdp-e?uA)$da(7fVo5WP?Y699R9-qa8b{rdAEx`uUY*Xy? z+NK#f5)sCv{Nk|huvH8fos`evXP(qYGN`Ea7TosW9WUaYU^sQkYNAeE7d+=mYg_B? zIK^Q|UaY(cK9NVRcj`PjnqdOW8l%Rb&DAaUSW^!9Rj;YX>xWq+l{)dwADwTa9gTEy z^prq&Mk?oaYFwhP_*C1l*nXB>383FVbOq%s(IIb5>*^Av&q~BeVSYDQ^dpCjh2(wA znZI=yM93vJXI)Ik)GJ~oA;0YgnE#{1!RReS~SDLpeq=EuaCE(>JY z@_&H=#YLtXxXLh4ay-2+ub`B)VjPs&nEncTgrQkiXT*5>1NZu=TNHU+7{eiAihaF9DZ@C^T$w_q>!V%tX}-L}C@8g1OCv(o9zQy51aXLAt*xFfzh<^OxFqd#y!*dk zrQOdH56is2JJyf+tYs>j4abh(&vGVQcAekG&XTp`p};VKQ>E zzEd7nx8IhH1wZ$Bej?&^dXFN0Up_i|U6Hk~!w~%sZ7?xjOa}$acEB3eQ&kN6-RSwe9wEoas%b6?X zXD<*q`2N6##g!+FJoCgljVB}VN-32JfEY1z;e>NF|!2z1p<`CCuTZ!iaVN8ae@lTtEM5>fDM9!S}Dl!_%OF<_Cx2jd0ksAVqDFyKZ;~CxsU`@l}mNAI) zXQN%T01;1zmgM&IQ7$d(k)BsXDIU_?4w^E}q01Wc5-9(V0Ybm8aBjB+rGlWW>~3Cw z%k7rTkJu=!C8!Z~VGn0?cMBcCJyt@o@&DxwSnuR{7shSQ`L9njOIU&t>KWTqOrsw~ zQh{85`SCB?RLiqGpR8L-5lL4_Fl+RnoOlSG@dVI!mnnI+7k5Y8%1pGkR{&Eq zNP;X|KHwTs$Wr{N*@W7)>KyON)<&^k3d)v+*n7YyckSs-nU3yjglug^{ zNoO;}K@Hnfm=~M5x87_}`h!8 zk8-g~bd;Nhtv^NHyS zI>bAui4q|Y1|iXd{DTcfj|VqAxp0{6TJ;V1X{pW?ufYksyxegiE#FkcVr`1+lT2NL zQG#R`dJ~i?4kX>U(~m-S!XOga&-ILMIGAA7Lcj0kWc#R2vA&j35H1}SX85Qk##!Ad z+4BFVyNtHGQ<4*)`AYvce#Tl)^S>_yGyiXGQ?5U&rO}-dAk0^_Pa~@kYFu^L)FwMP zFq?9jk}S14nvjgses<0qYPOfYl@IIu)zU{U8%K2DFV!;xsM=^F^PTsyZ{xQI<}mCq zESB@@nIggoKT^5ZIKyE=%l^2eH-585Rb$yFTW22+ ziGkxcTNR(^G&YlpIL-k{Z8*tIFGsUzs%9I#!_@PH@}^118O?K_uA`t6HM{1@pfeB7 zrk*b9T*OHWO;kwU65@Up$eWlXbGoMV7;7)EqaTrLCNE!@>AHMlssO+REmf! z$p%XpuW>yFnQv@>&JC%4dFT}|{Pr=Mi-rHB>Az!6*(2QMnbBp0v!xr7t*@i##Ct`* zD6=uwXU*e7%j2Ezdd^)~*rnK~7St78eIZ4nRQwE#OnPg?G9P+g7=7q%qqZ%|#*;E^ zdf|KbyBIrSZx^}AWHRRRyN3AbIfG(zC9TI{45NCd;S8f{scUVZ2Op<<@J=>B*5Bbg zlx_o5iv~-qZR9?Qrhd8^0zm^?<_foepg}5QI7YZHj<*DD1d9BGwg&vqkwW~Gul{{G zU?sC1#m?6wa8Sm_>aPn8-EYa^@tQ#|y+w&5S`TSac84@+BHUfR(IbchG7Yol@Ww7SQwq*q)ap z)QitgIyvGw9*sQRP_>p#9l$ss!Axsbo)LSTx%dXpe+BEReTVc13W(m=tY?tMC?3 z4Oh8+X(|pXkioEF@hBcX^ZI%3h2OQ`+|h^(Pm!*Xv}}Q(K?4~OH1PKnr}n5sSd;gb zx$!10byfQZ8a!U@FJvzCbonjx9O*BZTpkR-iUgK|{Lx&%lb|XG|1viP7)r8Ffj+CS z4cjT+1(7rs!uIrHaV!hbK#o50qNZG=QaiwMQq2U@LG{J`+3x8M$eSz($uf_Y3fReq zfWEjSx0=|S8Y^GjmGnqzCBf0i=qt9N51%G?zi~nQTDjIQwA98$X_y|hS)SA5rwX|f zKb8wIz$Ib#`rDmco(-D?)b*)mVQWHW@8(BL%mL)f*Rf_p1oEX7m}KEle(M0J7t zRXT;t{bUoLp43j)r9(6m?COhBqr&KHtfAE3>oXjb`$Z|LcTw1MSKzr|s610c_os6_ zWLoIHQSz&I{Z2iq4s^-iI^hmRkG;MjlMDW1Q+*t0O+;j}#pO9ZECp3XtROtRzXCMgerhKgp_ zxeuUq`^u!fc($qDT&wqAZb^E~QzCP9Z-nYn8g+?X>G-R;UQ2#26JRuRX&;eG@4BoKb?s5a%<7MSN znkak)mu@HgyRwhc%kJHgXP$`H;;FIpL(^6Gr_9@)nsPIpq(pll4gHGEf_LDMtGzm- zlgq&BT9(@egtUZ8_sFO-7ODCI0kWJMcvHgBU05+yF}JjAQRtyW`^~|;cRAunrGn#M z72P-VRDq%5|NEG$qq~5fy1g?%&EOj)v*TIP`T%@7P2Zm{ZFPB<{b%O&#S64Wp4+i$ z9YRm-{_o)1H0w|?5?>Ghtyz@Jwby^mqGWRa+5OuAIOt)%t4^D~FIeZCertk;L4Dts za;W^IqTJikl!F}Y)1{$!1@$M>&!L7E*^HN!>~HLaS(VL&*NvTS{BAzs$E)^;X2MlKr!td89Kk5!?X-5S5qMRDM=ve>YZiohLB}CsD66$PR zexYuMga01BMd}lGU6IdP@YW>hU0`v~CEi1W|H(4X6ah+VmX!zBgBb*#uT}RQJ=g%` z4@b~3pW}pYhA9gS7K9C)NR6J&gbUFOtsE9kW+H2Z7=<$Uek&&YN9D4n?TQ=Ie@FvQ zK%c7dAuF0$uuLB`TKi_28h7g7P_4A(4Nwi+&KcI}owjpq%l%uOJqp(u(hnD1p%>4Q z8fW*lHdzG1@QM^A@%ghyiRz&JijKHRzs z&cuFsxjYOktmXQDY|V{ue~e&_`160R0S??pMkFN#eeR#mC+=4S?+$YPk=6WSzTRF> zp`PAGevpie_yg>Nx4Z(Hs(e*?0+;Uk`6h9u{}Nb8v!y1_V#6b0dF`6CPJ9f;Trg79kZL%3LAECQHxLoVr3CR^Aw!c`h5)Wa^hb zeKNZfs7q*e%HXmnVc<(1{REd`Q1%;Gh%)o5FOV6`e4n?G=OpUX25m|$TC>%45PZKK zOdtFmU|Jo6a=y#fQH;kPZ~v4`*Y~YMJZ*FyXU@00qgvpZwm>g&DR${`JS3Gqv?sL; z&k_yOP@ORPZoM4$vZ-(i9A3WSwVvOPEk%^3=+%*WYA{1$)oBbBV5I4DSr9sHKLwE@ zXKxkv0~F#A%yh&u<#$4LMfBVFuS-J}jp$A+3H&Rcwp0#eNAlSPv=<+5-hk2~*R+0> z^I<)m*~^+g2)SfO6UF(6f9*w5xQ9~SlmC`Ta=^iWaeR=e^!9QwiTH_31YT0bU1?gZkrI^|S6(RR5c8@7@dT9_xm&&<#td9BPxT-ob_;f=7Gc(ZM;!8M^? z$lE_!$xky)z7_!_JnYWc7S2w;k_}}9izgGpx(#IQ0an({mk}ncV$)$tA-Sgg!OJ- zGPLwkH~7iojNmO;qX%&s`=vZuFz!e#WR3un_}L(lvB7|#7PRv#e`qX~3;`(_ErW2q zn_%-OeEg%z>i$3m4cKMBH9H$MWzgHa0rie2uXUd2`(7t_-ih?@`mw}>x>dN@u@egd zaw;j!$CQUG?UKrqGLqZQLJ9ZJSms~OvJ-=L;{Y7816G8|X?*rSgmi9wY`_3pLXCaO zOYbI9vIS`?JKGNi6%NW1C1K4=MqkhllR+N_HmKyGM9y!-tRUR8A|!X|D+y*5;S+gpr{G%cS&MY2C^$t4$X^VpJEh%G{4C$nWrEpE+1Zxv&YoR26T zHk)u;i9G8$lGHdS>@So{juPhBPGxAz^a~MFR9vT&KZ4b6tT>6NUTc1(<{zB!`4K%C zbaK6*#OU)nP!2&$B5nL*4dixy+1v|D1Jq(lQM#PFB_mxX5a5hZA=X)IFVjTX`Fs(Q zB(nnzN-Dlu0Y+%pJa(q!@=GhlRgm_11fb{V4H{;-*f8kmB>58vL?;(QCSg96R8;=f z@}tY?7W;>zI_<=4Dl8H}+-%!cZSu_j_#|}k++rwB^L>S33XKuB0Va0O2;CAvV5z_a zMkf775Q{+0fST(m~bJ}A5tORq1YWW=61W$pb<=#n?SlSV!W!O z7^f%GIAe3}B_k;iK?5BgI*N)j1A~%KMx*r(ahPp5TcQ>kRNXwScesX2)@FN&JsHZwVo^>^~Sy))rv754&LvDZrxQE)XJAWritk7j+qEIh?K z-YjkEsdhd9_q$O;j{vRO1p?L*0YzyvYp`x-n1a;48MHbksgjEMR0RAmxAkRm;&(^P z7q}rNR?OW6a*XmxcOFY{!{u*dIW5>qlgpXl0VAy6OfSM93O32y7!6co|SjwvHUTxtd zwpDEy2jSh{7u409< z$OOlXZXL}thG&b|#%#kPzGh>}ZeG{G2h0+rz8_VKNaJufCi5L6q|q~AxZ^q1ZV5go0G0~Y*{Lx4df|ea#0{up9ng*02X8agnGaii(bNOoy(2~4B)-{I9&&e?=J4meQWq*q(CSE();2C@Fbw+U*(TH*$lUjqi1_KjbYkoQp*av5wnJe>Eko z99q<}P0QXUdl_-QPev2LM^A~XLjhp}T9@n$ux!~grCt#Ogh-~A)kXC6g#cI5N#*Oy zk~O^w2RmDbwrwp4S2%M{uA6CY(KR>!-2y5La@iO#EP(etffUYg1WN>K0WY%^^K)Q> zSE6XZMO_h+FW8u6N_u)B@+#i4Dc?_L70)8b+QA<>q=ufPF!N&Fu8RS49q|_QdZvk4 zJL&qd`Ry2PG(~Bz3mYFke3a$|@(4I)4dAx90>oG@#R!xS=O?`%v$CPLIrNO|%9dK< zSN-vQ=`Ru|b0Y}RGf$$+N8N>s$50NC)dCX8l>FvO$H7kQw9_WR-e12bHg-8>6Rpc0QOh zLD<%y`JtVxcrVPFVP__-VjJQVwk8dDClJv|C?B5&c!`$K@Tw$d??%h3xxH;?R9wia zxw-9RR$O?@sG36Ml#k1=u&S;7z1~wbHLs{|xCc%HEkuEnz*pm3;2x@`rC-^=9_Uff zc{~8epVR-l3($DF_kIJ-9%!KVUjyJ)|DF8rS7Y5F4>FXue>q}-VHgE}Wkze%m!(Vu z@*823G`6Ev&E^HzzQ?BtfU+fgHiZW~eXr*r$ErP)E&Ilg9+g#;=f zg`vDX=s$EodQ-H~G7n6jQ{oA2qQ05P%cdTL=3FoZSgrg*5v=EA7d}zn1O4<7YyDTb5;WH1(Wi+VYRC1bs zVLn+JP24<`Z4RPxKo(u~PZ1=BwWubQEfY+#5LQp>!Xt{IE>})R`eQZMh<~Az-HOn% zKnu>gfMiloYpC-oQ-)ZLd>HI)bq>rwk7VzPhj#U80L=Iv1}TL82-pB5X$vdgh_V_< zTJhE(YW> zWzed`R|m&E-^;feYv+C{OfXu21#5gwMg?p|gI1u0F*+)=vmW*%-pvkbq zBsZ6-KSa)f09Bxb&J~&a$U~>i|5C4ZIKVc*bqi>(5)*MH6&HEK{h5FPuB-+6K2hrb zNvdTtdATk#2OXnrm9dnV#$3l@4PFI#Q>ZgLw+c!HVCDmz$@$5+UNDfDw8eM3Y@0H+ z)LHnn(A5{j+ibz7-Z=nEV(t0M!=8YbE{j-mv|xPJ>LYn=jmbFKsKb*d`^3^hg5WRx zm9g<(U9935@s?$gfq^%z-wWnF1zOE{V~@o?q#9>k9(pPDHqZQpVU>Sq9r?kNcU))_ zY5z+aphaNV5O6$IQVc*sW3jpgQu@Kx{# zhk^4g>85c{u@ss@#8heb_q*F(cwbu+w1gIlk55c|u{aqEYAi;XPM}T3m$5gAwuT_m zy9XyF={dA}?=W(*R60FHZQZQyJ6AJiC6gb(D#l}`%hpI)`7Z`@G0NRG@@q6bc(1|& zMlF>b>iRui6Sj@qp&ohj44-%p8Lq1<$WgzykMj;M`5cyX1$+(nUMOL!7LDzejYPOd zr;~E!dsFRTqb<(k=3`S8N=Wrd4<#EO?exKl8aDPa;U#Vjgrlj%)N2(6ra}gDKPd*G zB7q2n4QKy#PG67Poh6yH3XjLc3MZTj8o@ui&R}kuX?&IOExWA_EZ+W73PX(M8*>P= z)mZ1Wk@?XB6B8D&HM57QS?+D;NIQwE*?)L5w5LDKirAb4p$BmeYZK%7t|C~x4DElz zo7Mvc#z2RV!T|yTA{7|R&H1~gLZMb2l(@x3)PkD!_w<~IVy5-Bo$FgRdR0v^cHVl; zKhl6#n&6CY7u;Pgm{7WX_6$goOA{|`Hm7H zCQ#YX7{_gQe0<}r%gJQ$?h8eF6Yk?66nirzD%%VT-Y?0} zLhyOsK>~8xluW0gSU6D#&dP=*%v|Ct_&77PvzoEZ^aJVuhd#JXRXYI=A5Tj)gUN9~ zhT4gOEs$wdeE8$7@$2#w=x*$iA|Ck~T2&JG7whiP`=jgs0}YDIBe$?8-#`>xj9I!y zX%U(lza_2dC8$O1Tm%aajw(LJywUy#G{CstTG|SIKw7fnF_Bz*_^feHpF~p2cHI95 z8brT#b)hq7j0JjJLgUBR^$0e3TKIZv*gAB*X`E#M(^RQrx}BsZFwRlkHp-<>sQ!uG5XP)Amv)HsGfKsK;5NZ z0LIj>=&8uxNXXN;MBg)*cd#)OV8wH0dOij|s7o^HHyEe^fvg55xts7j;o2Te&Suoay;m3n z;rP0fIMRwu^;f#8%(Ttd*ddvQ3*49K`)Nlin)I$&Dq}$SSo&wAM0UrAc>*Mc^ugWeKzc^Zjk&uR2MA-s{B^u3u@H3l=1|W zFMz*z%Jn3Lc>6#hmm#VSlMG1NvSo2?zkWdha!Cfa9XMxFp9t1^e|2x z96;cqu4+a(VzwO@@(AXOpYUhv1V^~X&*>p3iG@$Ggy!9&f|Nn|h(+ag5Nazfmu6xq zM!HZ#Cw(c!1%cDp`g!Py_K={27>CRx)oNnRpD9HR4(IZ%f?fH1tmAq*-hR@tu<381 z5tmSUbnCx~R8h#u^SX&IpdVgWBLZC`a){Vyml=M{P<0cKWyaIr*?fMq#e{os_ zcV_nrtOj6ZST$GG!jE8&Z~DlU)S^6SHJ^ZA+aZ#YQJf3(IRha8C!o^G*}6lPd{3)u zy;yLY23M>N*Ur?fDaCg<7>CvzXmd@41~T@B!lU!0rv@7%W3<(-t%uwy8OK9_<)A+y zF3HO-a#j}olfRJC8W9t84m>&|X)TJ|lkT@X3qJ9St40qYZGP%%kp!$s(4V4ou%dd$ zZKu{pyK2xo-7y@7nd3Pe^tR#gRx!-%LYaN`7N5dNu1UQ$$G7VX8#W|ad&*4rct`qx z4I3fqE*XOe+qcrrAP&36!L?K(zl8!UQ-o>to~0}qcw6V^UuTEvT;)ExvAC|+K3)l# zhi8zGp-_)Yxw#ki@Qxbi?=WOXUfmf*pU0-J2d0}g*wKD;j3~D@7+!AZ*Q{0lW?E^a z!cLJ1V86g(i1^j>81|#{(q-(9q~XUohvk5O1LxV$Roc@yRxSC|Q|lWDt+zNT>=Ai> zs&+!3avyrU9MF+cSh6*O&S}GWb+M7_`7f_h_YF`mbs<(!ESRYR4Wyr+ zAD9Y;J<6W6S9JmWVpvs80gKJbEpuIV5maSKvM;@4LgzKIfqUc9={lk)?eUm+tEbF- z`DArNs)Ws}VDrkv3oV#192RM|`nFX|<1oCv5<&h zJJEEeDPuaL=ks*e_k5}FvTSJr3qaKf7&6m+h}R$=K8vK)IH=W2G`y=VXUUhgIDI5X zk`tjs%%RTg%D&KfRk90XEDzPnrT3QKD#99c+4LMJ)h}SV#NW#31iwGbaS-52HRYx? zy)E5unKD@%8?ER@xagIh$R(0`;c`##$CRt4!V@Os^rPY3{I*NP6Nkij@Ow zfyz!TUGrW|o)eMY$FQ~9tQh1z`uBI@A(0dtBJ@_1_QQ)X6m5o92Ftec{DwV>RS-#5 z{F7V^T22dP>^z^;U;MbW)!yVwun{PGiwUhi)<4)obUy*AvLJi4uQ-X1-Byb%R2D@BOC24j{ zizNz;wU{W-GA-^b7EKZ)P{O)F8!JFplvjl>!A{Hep_1e{u3Eb9x0jw}Y&RuG>wgm|(2=IFuzcJ7- zdvKFD3?I&wF^f@}5XW96q)#zJ^$yp}(r@@w?2Mc64z#Z})QdG(Fnpa#R(DgQq zUYtTW^mlN=s^9EK-SCf=Dbr3azumM98z^W@i=|3I2n3gG%XUVLrZEC$MI|A=bK5Y!zR8Tr*^791E` z{g-J!Y$7$lli9xvVDP9qh+dD6I1?8&=+uC@1@Ov{f&2z(gTWdmYHIx|mA~8x4T_$bICk*Ki$-~vy?E@ezvGAz;m2zID$VjJZ{_jcml&+~D+iZWQL|E4Yd(h3 zt<^9m6bSyQOk!}OiKW;fTG^*&`t_~jJ#0{r2jONfBmPXfa z(1cvmJdmrzvj~y)O&LcMcG;MD)p4OV-Qro#v>=6b+RMwkf7+|0U;3_O^|D)YrQ6Cv zw`f;sn^7EAy1})2KD$QZEyJ477?zdMYM;WIR`xio^)K_Zo&9eHnDZ;j2Vy&+TT` zNA%vN`>ots;7@u-mQU&iRym3ZQQ;SyW>LwN3WI%yf#sauBW>wdd@(D<^54`52K|jl zpX|%YumCcanfG{jIz@WB)G)`N9(a;^OT9P7wvLJI%;k9hX`a#-5m?IcjNdu`4b&;P zf~m9=1X0gy*bg8RiILfM0-7AIde+F%mYLmWMWy*x!i<~q-PFzOe# zmIAAvpCEu&JbO|+gRrJ-E%Md4Z}aojY@FDjR$IBFEFsQdJVLVitBo zb|LvjuO#POKM7paEEFNhzV#L%z04FLJqRQU2vPVN*n9z-d8nsSq?eTYx*RAvlBIJO zkJiQIzb+9dYr{MQkh<`I&j=;?@aA9n!KYnWh_+awsRIU83x*Rzv%*eQ3;83!-ZOe3 ze_&{KYlT@poB+6>gnRif@`=&u*~g*F>FHUKiLCVS6)QE6=qfUamm0&~=#d&rON#cT z!BO||wpDT4<`S!oAKyb(;csh_fA4#rrT|LLz?8oi==!{XX3lG@*qU#k3k!Xa6@7xI z7h1aS{>Wa@{?nSu7nTuov?QrPh$Qe`mmrvi#o*Qc#P?F#7i% zaogrEfqfGKp3Bx%E})i9hPEq({L2!#t=)H90iY0Jq}j3Dw%85085ZGLwt(s4yg{rz zxEGwx?O&0jiQ}%5SB9h$&S7OyRDPGfU7AYOpggn^#!!$opAsy>IlZq$BYoDL%dJSr zF3}*c(%Gy^r=-RqgPZF?4T|hZr2@UUIh< zsMTuRm1vDwK|^J)={c}=iS{dox8ul*zkz~zv`Jlo35uReF!lOpm%p(50{U(5lz3eg zOXuTa)@+b}_;CW7)0DWE2u;qXtNl<)F>N`!86x8@=6V`!JHs#YxNp4))T?}+hgU+! zJi5~)-@Eu~&Tiski0e;pxKOG+I~P7NW*|B3t>=5{A*O-Rdc_gL?kOq0a>qY-Cg*)? zA)D8k1JfeVWz8#X;35 zlap7_AWOoJA>I=zM6)$w(`&7fGsR{U?v~VJ=^XMSGWOuMw}spGy=eY8798B(Jggvh zcG14R#cA!kP4Hg@WRF?hL|p5dGZ`KhzuMW!_!iEcA^UOv!NgHO47X-EsAOExofXA~ z{oJ#T5M4t~CN)WZ;`Kh8iFnMI z)7HZo-rY1V)Jr2z^%bN+O_s@(yw*XiIT(aA-?_d%qyeV8;5Z`a`fZtWrtm6ffcAZ@ zlJ+Fr!6geFE}V+WGMhs21dvR-AHV?Te*X3gwHMJcXSqK_Y(5JvMBiUH=TgRNC|zjo zUcVtKSKpJ%mUmsx0keuhBbC3e(C&!UKhiCQh_5f79Jd9$4ljAIJo=D%Ny6#p((mXs zAu{FAO`J3HQi%lbvIHeD<3%|Zw`t=gc_+ptRx90UILuPN1QZ&o$j*GJ5E|+IB_${O zWOQ7^pzQ;v`ds!@w9mk-uD8^#Y}g8R+99~R9)Ay>ch+ox3(xz{hS#{Bcl+FQ8%}Gz zxM0TiA^)@@Y@cFvm;>M++_utUbG5<4C>_$}OQ?)s<$a%`yT9nS)R}Q%|`) zyVy&9H&JxB1y2kR&RFt2trtWh$Q}dX$H^n)@J|G7Z>g4K5z>fC&e!W0vfrfiKH!Yq z9kjn-z0Hi=afvzQoU!j0A#T-RIAZ?=Z}_tB4ZGIO7-0viOd3?nwaET-83a1%(=#$o zMO&iwD^2!S&>Sl492rjGk!%AmKjn#oi4EUWN*dQ!K`KvcysmENrKKdW)c6Ffm2?uF zv8rn_^fxq_x*P*(N0{II%fMsgAD`N2C;bs46d&K(0eYgHBBE3sJfUl{rPcl9m_f5U;XvI{C>Uoe%iCpvGD`KW7$kz#Qh@2_5EL>Xdjo_yJZgzTNon5n24jfsL;iht!trDdy*3@XCX zrj0Nbs*9{uGJ34iErx>QFrROH_NjZKm39tK2eO{Ge~i^xGXs`V|32jU`OH7;dbPUt z!Sovt>Mr4--O*JAny76b3o0;rzB6yML4B$Yh0D#SdU&y<|FAj8&Ly4w_o4(?9C&E6 z8-|9p*~;KY8=&|jwy9g-71}%3Uxm=H;Yi6a4mU;RjFQt3P23cA-+&$hx6}(}FX0NvSr$}yJ3*k2WU0oSWxOT|xYkIiGOP?C;)25Tw zM>%9X!a9{B_H;8pThdR|xgdj{Zu$d+EZ~x{a8BOykIQgQv!xVS%JaN-dtc$>@7NhJ zqg?F~I536o;6}Rz`#WZj(c3HF+DUhIthFT7OCI)ZjNIXW-uSKhV7^TpLc0%st#rm6 zkzvdw*D^%d1>0P|v3KqfUxh;~jlBg$K427D4vYH;jqfz~xb&5BcO7^p>v$JU^$>0@ z?-q6z<0rIYEjf*iL!F3FH0F(^!I!5#fpJEcP}Pef&(6EqIWR%c+G-s^=-|E{r=S#D zmP=`m7t{usfB#N|l~E0eJMnjG$HIv|bpl1p;41NQh$fwZlgt!{APJS!dGSg>2x0%l z`oI=2<~Gu~zQ`9Yj-BO;4>jZ~R|FE>j;9{9k(OM1A;@4dIr(!phs*qE2pej9x8OuP3CZgpnOgw*G4D^68tIhne-~JlLlMgUU$P0?cX}KZ3~WeVZBbH zf6kSVz7r47{oJYR9JTQOP}nGa#}$c9BBblbhlq%>oRa_Q?-t!yR)%pc-hav zu}P&V??i)?^Sl0R{q^+l4(H_M7e+MAyXx2n7a57CFD;Zx#MyXA)UTiI=YRY6;b>w7 z3oEzV6Yw+FX8vdKTQAaqq%1|^qe=ahFGZrOy-u7-J%e-`ez2Tt?DfhMTt)-BM?Co= zg2s(vXpu=J=;lDSykEeIFJdHkfG`6KO^w1upbmnb>~Ubc=Y37J(Q?p@%yQB%BAsZw z{K-{IZlj2oPsUEfQI?TS4XmqQm+#~4LeE~NdYyi+dymEL&T=cuF1EntDZ=l5MIHM$ zzX~ZnMXEi6&El?G*?PbR1r1mFFfSxLy<&12*!vQuH12MleKbXrV=^Pyc3=@Phebq= zQ5@I|x;G!hv**8V==(LF%Ej%jBIQ#fK(=n=^B(U&&arO91%3OOd&&9)+}jg&KGXTI z2`(Ia8WR~XeBaS3grz>rEBlj#p@cHjh(MG|I&9yD1|fX7ta|OUl1TR3Ia#edXFpkO zf|S9QFGZlMLm8U)x*tX0lE0sEC$+qRX>oVUXa?KSpj@j{0BPq4| zU)K6UWYV1*XBZm0>h(t}zDk&_>Wa!S8%irS_p_EG(&@Q#RX-c1Myn>aGHb1UnETxa zu_^3#4llh|KjgN%x0}CH|A5fb-x9aWEM?!>S-*jL$n10ZQ#{X(dilwv?{KJQGQ3(W zx7YwRB`!pYgi@YL>~gKj62T1U&EsOM@N+b*P4yMmlqtfX8;(*S3alqETR$s-qK#3f^-UpwFI>^py z^Z7tlm_ZFFVt3T}dPCyv$i0~XghuXPvB6>(gtw`mMeVKa-vp-WxSrMaR;3gf94V)8 zYbiQI7dzCcUm%|QI@iJKnGr&?y){#5rYjEB7%0_jk8Zr}%%(VH(E@q4y7``P=gGDMVC4#d0TGo4*{QhC}_C zXZWTHpV(u!ZK6{Lqfq@?=0F1>Xiq(RVJ-MIoo;PInMXgalUgAfWU?LKZ|*H*ORhgn zYQGnLrJ9?Q8EZEvXRq3r7wUz?V7JgsK&us@Qq!uDOx&bDoc%Uru2&FT{{kL_?@I}L zR#o)sxqNWVZG)x04?OLHIg@g1c9>L*3uQNqq|q8V?t+O1NrQd=a(xZ%-4mD2@6S^D zsG(yd_;Ef^+KsK?lYB6^A}PP#9t2I0kx~+vG4zpYTs!x;`L^smD!@jnMwvHlPzabx zPdYRA`$BmrC!&|69Xqp9z%`U~r(L3LGWNs|MI?>+zO=#TOyv5uzpVKavP|ug|#lyk1vzm@0EJ&eQLC@=;_ip14&&$#B z;L;2H72GI)>Dc=IL~jyC`k;zfxSFjG+6-|?3~z-787F1uD3{Ebk+HwHY(z;x(PDOj z7Z`~ngNk@ULi%|nzxj+?=?I#h|5vB^UHi~d!k##OMz_ZI4^4m^m<>q81K)`@x8YZ< z)+&Ry%VqLf7fhHfpOl2>&QYrxHuyHUwSFQ`i38XwXcpJIlRs>ap3Vn|R1O=2P%X_2 zr1b!sG0VAY+trfnBmq72R1|wtvEMAFIa(iRK<^mspIOA0zNcNbi9>^wHi!ixS}W%Z z#>^lQU6h=YWJo{jtJo0O&Lif~&bT7F%)0XL68uyRHkRDmCOnM4*5s6#`a>8vk85~= znaSyI%Nt1j$GB>Qh^ut1E5dh`&v9!4)qa>P+3pv=4SQg}|8|CE{eu?DyFG4pit znzCuak@StX^G!!(Kv%%Y7d@%+d2IbWt1B-Xf zkw>fmtZ@flPmYoK+*KykKo#6iXO_-an+Ei&JeNVb@O`re{CylhRt|Ev1pVB7e1KWX zoc2B4eEtAZ!OvGHrK6RXKe=&!27mmc$$+}&D*}O?>|460>2&5?L0Dp`{2o5Gxfghy z8aHup%Mv4Z#7TDr^V^q;E$CVl_EVsEf0t~%>6~azqo&S=%@Bfp<}dvb@V~hy%X?q0 zY+4g$2{klr&<(Fl+npb!bcdL{P!##EEVhxba7yRW%Wu~H708aUKB@bLR>_ZM@ZQg} ze=!}At80$OIDlUWY3KQ8Nn3_A@}_fCs5GRD-MX$V+-jUH9J4I^mJ#eMf|(3~ z74!oBL8$NnHG!BfmtW2Rc`~(?|TQcv-)OR7>%o@*J?; zR9!KZ$=>MHH;%(IcDqff{8Hw3DsYXhMNnf9TR@8}h1LMP&V_=4)IN;X-MGlWgWvkQ zPBC-x^V9GXJw>*5gW<33st)tup(7C^YlW)fr8huu=BGcG_xsz{(O9IPBVcL)1n3blH2r@L(rhT^4ZhVt#;r^76H!*;Y|g^x-dOK}i}zu?h@p-4+M zTM0qxG4(`ekXrA(7v2Ae88||Tr_QCpaIi?$6@`5p>=yVzn9YP%g5F3yaXELO4;X^a)4@pPnZJfE10AlYHlObCt_k*(7@?57dMxExveXUI6Im-tF zl(uQ0W$|W?s={b)pcgf?U$!H(@Ylf&_@oX~?YbUpQsWqlBwN5xN3Th6?3hy?^lHrn z3>n*i3zdzjynJL?ebvcj4Lg%NvZ~GEH(Iy|0257a>e>W}!9JJ~R=8^Ewpepfs40 zSZD(`3%q{L!Mw>4j)T^dZ0t57BDBVmgLZ1P;QMbWtvvdJa-lKUFsm+vbV+SmzpniF zr^p+^bVA`A%N*vX>~!x*RQ1zD1|xNsUx9LSng^r*H_E{zn%?9?p6*4--U-+p>#Gr0 zh>0OxbL8n}urnu3hoFF6;HM!N;R)Z#=VK^O2EZXs1g*OL5|FR7 z9LVMFDc_^EcLnPRVAgwq8>AGGHjvc-`M?M+8Y|fUEpqi0Fv^oe_ zaG@3CE=`_U_T;uApAqUpi~a?wppOQ#HhW_kE{p>G+O+Ji|3BPOy>GOnGx$FMl|X90 z-v7d?tv+!>mn&jc;f7`mVWX%Z3`3ZEh|1;8ZFxd&*HSU7L)zS%gKM@qIBA`b9*XAW zGjY(w)wT|0eNRDp)dBo=FyFagduRu)-TOv6kCU){0WMN*JNV+Y7sB~POJ1vs<^m}j z`Hn(%#~?d|yCG4|DvE76u7m8($3iyWeiVi|+I&r;jwb9ahO@ficcxV12Zc;`UUol~ za#2gZ7J&&H>6qcGjuy;zFvB|O@cK4k^AJuhFm^u0tu|=xsc!%XS!P02C2M4mzE%#E z)l54m0%U{2XmPHcfKxJ6NTK+GXI$4NkV+XnR~pIchNr!0%Ji13e&R*J(*AK{^tmB= zxMQVhZgld+40gL29xl5MI8^{sGG(dLOHTazV zLzXV4+l?}*l$Lp&4}z*WaNxPk-7i@p^PGvaQ(?FM9^5Vs^KPB1E5e1+jHVs&x|7N= zsT`Ba;Xjkg(fdp)2bolkvHhD=j!EU%v&wNMG%IhqTqreY>+hfLj-#!KI3hkynqo<% z`3O7l>b zBlP~flv*aT>)7okt`_cPvvUPoNzJKd9%}@WAd_mAWnG&DJ+3G~G~`rO6tdKag5pW& z+}s~2W=Q_3o#m_G3vsFb<#I=8bcU~co55hklf zSJ{QI{qy{3hv?kP;M?BMi`HVar4u(zHB3yv{az=QnMKK&E_tHnj1FhEm?)9Jb+%n_*XO`5CHjL-#lIKiTp;Y7sm`brkiWJW1E#;b2Li4OXoi5#) zXx{AxmsN-8`PnfqFdl@M7Md8gw9ERBctb0iGtcMN2}HPM$-pb>#NzR(HAMv4SPRdQA#|d}AB#yRZ;&$(K(dhOoLtD9*^LOxqYxAa z2K!C5U>U?F^-;u}dk|M|4Pm2nOAAEeh=Qfp{A|(Dm}$6A>i;OnGu+|w76QD!K6_Og z-^{mPUBI-F=lGUldc{++025;mO{3%=33dz7VKtDR+wr!Wu8q2*wxQ!5qQA%X3s&YF zA%+*+p2zne4pRiMC~s%w?QAxiwM3Ubg~yw5=#7w?=5#hJsmG$gPqHaCq34! zwglGkSqH*UWw-Z$0Glxz3Qf-Plf{>N-o#J%W?w6Y*<=Q5r9D15Na(xI9| z0#55J!sXPrtx*$04 zQ1!yz-zU}tpVOjOp%f_^W>Q`{D_bMLl7si+gOvI$SD+*x!21WM*)83ugFeH{*%A(j z2XN%RXmIle_IwZ3zLVhZ`61Y$UVBAZiPLC~Z#au#d|I0CY+)Q-Zm=|t^C#4IR#D1? z_2XoGYt*CC2#|SmfpN=?=HxBi5Sl6J#?Hm|{W>?ZJ-}SKAPcG2JJ+&V#@&n3Gwk)z zbZgqt7pN$pj9Qqm)aa2mNRR3;zTtgXpug~s#Oq*dypwPSA~wg_E~rei6~GK?_6!J^ zLW)@_dpCQfSc%qP0Cw8BfNTIeUc5Y~D@LexJE`v$lP&il*CdtTTdlkG?b$2xlpL9U z9m5t`QVz)2O4G6ztGP8wzI24=^|eZ5NtQHI?8GoJ;89Hgeh*c5>DLtmqEEvBMC!dy zL$mMuDThwOkj1V)z87}jw~F<+V0H_3?Tt6R^5ko91GmmT?zx<43K?(6v)l{4aC7l1 z*tb>#*I48VcC^~Jvm^M~BVA;V#LpCa!$o!X9yUyjBN-Ih*YFWmZMf4`Z9u&ywR(+@ zJlro8cw6LTYcsk_5Y9bC_+&;XDXBD%@M2=@Na|B^q{cgJK_9Mehw5 z0l2D{4y^*{>fX|qhn#W#-s$)M#ULijq13H5BDTzI;SSe%^YO3PcX%bP@0XimtMBIH zU$O61@0GUGcXRu-`wrL9uhsW}Cswf}k$G`m%4Kg>;BJ{6{K%#yxhvNhW!lyu-H%ug zW}|y@fnGW6VZ1>mG+Rrh2j^pFPPRYu;2ObS1AhxAP=PHC-nB&^g-jVyYguKfnF~}! zt>+Eiv#|;TG!18NDhl~@wIsK)yq-gTAm;y>K{1gIM-~)RuTN}r%nftq%xe_)50ziQ z$n6u`-2tMm{y@_kxXG4y?ztidZ`>Ro*mvH&zZ@RAt656L#wP6nFIf!?7?3Tz)U^>RH*rWD_IAp4Pr8SJJ4FU9nD1 z3GQrAhs+H3D|v0{@&s1fM$XhZ-R{>zBgXNZmK*Z**{dPjva=H&@ns_%9%BFrx)O2* zg}m0hP~l=ITN*iNYA#>0;%UDo*e?dRBzGabaG_S2DCb@?9AbpHIyA<<(F}}zXpGQL z#LPx(D!*YTgxxSfc;Ddlnr$$;TN-5`$*R7yMt3l3u7;amldQND7XY3oJurNEso03w#4N4WuLDIAggOLS4Jz5 ztfNJ!0rhS_i`g-kwaqXBy-52P*_UPWc^xueYQvPTA0f79Uz0-Hq?S zz_b5;JgAz^Y|sSoz;I5xrN-HDzQ>Q;4U^Ok_5Ij#oB(AGCkR3QMHpvrbE26otu$x| z?6bZMFp#hzaMx%Hyr7rnXj4mO^w;|;Q>$Fjm!;{L<}|w~SQ05ZfBp92wEZFQC0o>r z!S95o)*m?}JF~@F%4>4uz(Q7BudBs8k-3K(GE;n|p4dG)GrjI(zXYVJtgcyS3&Yms z>7Zjt-E){u8-lYKM6$5owg(OD3YO|YI?L`d45L~EmcrG8^=p^8{tpasdsfh7@SuTN zh1-Zpk)!#0^94!G9hAdT@}+ykq)Sme--#@NMBnWhtxRt{H}(sa;efjsdu;>?ei~ySu5= zbu}=eepTkxR(M$3g*9w68g_1Vc@$!T&YI_bH(MYx=svgGBUwQQezY08r%56!p(9G^ zUhI6(#Vp%zb@bR;RK5M>1)M^xXwN8m!ejls7YCY zD5nD!r@OCXfO2+Vak@09u|PR-usB_vpMY{Yx;p6@lb*5rOnOG|GwB&*(lf^PZ_+a+ zJ!5}*#+sIC5c$Qg-rmBQfUnKkq+Ho#d53r3F(@gtoLgbbFbkegbw%|vnBMq=q@qn#wKTjUeN@|sDx%v_e$Oh*W)@xx@7Q|4x->!F+DcYs4W;` z#6Bt#Ba(1kGsB#Uie}k{fY3T8XTo$ub)Tr74q$VPL(Hm$y@Yqn9^;~NsqplM@{BGz z&lLdR)K0{@uXA!K$v-PrZpbQ=3)9mRka{oIR7gb>Pgz3C9*hI~lKGySMK3`(*Nohj zT(i2E7$pf4x}=#G*85SA3p2LviQuZqoO6Mw*17I9ma9*nXquX%{OQw`(_C|=>8uZy z)o8fqvT;wCW4o;sgEsi-69grFLZXH|d}{Q@PWjNscY3xf^?NaGzkl9yTDzm!l$Ir3 z1zk+|RvlUet8D+aiwQK=`;lT9qPMaV`l4E#Q@xJKmw#1jZ!d_#&q$?Ampb$~N|u$b z%D(%cWErE14W>u;`*=YddBtNg5D#D5cfW(I^b6Dy@o>4B+=5;i2r=>kjCe<4l)Paj zxvDeattA5U;a_Hy=a(!mjQVobn~ylI`i92>^?mgUgHhpky)$w-`-tN2gs4bS;Nc`Pb6@0a~Q&mw|w z{r3WM)!ry|_u4l+1vDbCYsiFVZo>>LfD_bh^tjP;?D?*-OCWoR|HMe<8l;&TQdvPE zPoOZHp_;?8)|=!KMAbW+#R-AHO?3K05m0o3~$Ip1=N)AOHM=sPb$4*Uv{ku-89E5;;Ep z>n@I&ds?D77kVbk86KP$5au*q6Ce%HO4RM=UVixX2g*NafeWUo)2;-${*xll=FgwO zPW<}R{OQ^J*&pWQD_N3E-m={O1fTsG?${cBkqNrUTou2snP4SP zwno=OeaWbiM97M2cJFOi&-$#sYggEb+0IY*y?i?ah1g8A@4Yx>v%IJpfk}pnbVf5N z7&-s$<*@M%Tm@inZ-9$zHll>JMmB(aw`*q&GXs_{806#l7+rutS%(0xBJD~{^bU(ApJ0B`mei)e(HdNxY=YU{6fu?-N# zB5_VQYg;9NR`qY)RF+hwVB$b2_Nr($jFPY&LZp@HViQ1Ks)&&5Vxo~fzg970J znvhWm^K=CG1$Q(rGIm0=lq8d4h3KlM0v%N5OyZpCKJ+Q#Lerrai1mV9*@k+<@q8hq z{=#kNKu6t&Mk&3KJSCL^W27LJKwm*xLvU?K&XtkchwP3Qjv3X{sW75-k;&wmr50Ti zTx@j<=ij{?-uqTH!W;s?#4?J*%;C$K3FHbcSnaXrp*>p9*-5>v*Ax@Aixh~$L7>7O zu^}+sdXZ#3YQq5Qf1oFFPTm>4H_JB20<73R9GkpgM6iS@MaxZH;UlLTJH%y`VO~OQ zRe@|~rOPVO!@@Wvi&9=QL6WRenw79CJ-jrSb(+o)G7UnWEePR|802_UPE*)1-_)0b zn;kRLbg5ZsMq;LX!Fn^V2*$=4Q{rW%VPR zX84S%z8`j=xyNuI6a~kwP4i`wyXH%@c)(&x^`lk&1^J>SqF6Ci&_b<^-~l{<94Y&U zxm|?$^`Bsc+OH2~i@E2E9f|0u^K2Z)c#83<<#DvcQ$ydVxD-|TQf3(|VWMo}am>Q6 z{ge!_=VR#1N#lcX%7b9H5fq(tg^;46Gi|QjVgym_)0}+2wx*F&vB61v-Nlx9|57uF@9id7T3hH})0+s*6|f z5-GUy?AviWX{$$#o6Da?@0}idzyHKv1O#k%d;lkP`)Nlnou47bg*8bcKO0H4eR96xBCVZ}H4H_`Ogq^7kYwSWuiB6~YhI?afJa0Hn@6Rv0GC8Jw+8q7D><6^kC6Ti3@JT?pWyd3K==wi3@iyU~RL zYno>Us~qv#0K&m`Cxe1MK4StXwx1UNyT8n1n5R4GxTIDgojqoa*e>Qsw82oHnlnK(q`C^+pme0SJDM?WLaajvP&vK223}C zhIb`t3y-@n@-}u4v7RcCVEh=5+7fo0I|4h{4lLq3Ut%J0lqyMFsam%&P7vUzBURp( zX8Ni(g3EsvG`YTo@S8;D1=Th{{sd(yU+@u!hdL@1+dW)tOMc#P$&oiVI}%}xF5x6m zml59@KB1MCIn_J?dl_F6DPrUlT(=x8@#gqB*CC1lAf&ixQ#d)1ch|1;*fBlLKcj?I zT#)Y3ds0wDXe>UL=qrW%i;^+IbE7U%p-IY@_WDz`D|S?8cNW-ap5Lzl53<!A+Ll@<8Yd%_iepyPTY9xdGc7M`aC%tvjTiegSvfk=RHuoH2EfuwmY~En+c|^TX zcaUT0C`L-q7U*d~6!_gO-OL5THF2?WR`3t5)vrrBhV18vtZ&==o+PZ$k%o2RZ=X52V5nlWhf7kKQ~>N7h{5n5!-rF*DbG}M=UsN+ zvXLZ*@3pZ;aDvn6-EIGtRv&i=NBa=#5B;!bKdQ5SA<`MsTxN7;-NBPxiCf&*=#&QQGv!Q-^O@)!nZ@BC!R@{DU3~Xi7 zYY+!{!N{7g){b043z~3^p&?}&tDs}j1l5&l#7B@lzzM%cCvCmr`<49SB6z~2j`u-)KXGCxNxmSGe_S5DsZ!mHj|G2QKjSE1>fce;acnNl|N;g`8 zawmWE_HO|~JV6YMSxt7**}~Tb+ducGb5}gS}}Vei~}+4{u!C zTD0;gbFLlG4s8`-(5+(7c&1~e^JN_&%kGMdCtPP6UtkCA_m%^8Wb~#cJa=U(%;e5o zIRk4@ug5-EAYl7pci;rU;r+H)%7~X$FB;SK)7rf?;0FS@03rycTEyp2s?0AD{N88UsbiL6lI2Ii# zTDNOqCmuo8>~m%3ID%{&w+135F3w0=@*7sR5nF6w-1T6cH1T4ISHwUL3Aox`^h z!y1b6xUelvimEK6it95KJEsSXH8$7CrPg6~XWTh}Wq<@dm*#oehLf=_js`m;#`!FJ zcEDDtSQ{P}-AzotJv_v?4$li@)Cs?GSln)H4HFWE5pYE@J>ROx7OATCCQTxk$U;`a z?%_08sMbBhGik={a;fWvZ-($BTGQI3VezFIy~~ug^1kOYTe7l@Wdn;%H+LebWVm+B zp?OAG6}qQ%E6S9TbF2&vUNaUIN@rOA(!!0~S^VZ)H06vTHu9!RcCh*^#ynIlug@b0b3xX1tjZF2t;c@L?f`OwK<)~?$Gro`FzyI3ZKzSsLR zQc{Q>b(EwVR$6DU8ofV9!shnA5RInOT9#KH;^O&tFY8K%TX;!>;Q0hmL>!=!wOcc) zI8&P3(v1VBqfUfEBd;sg*+a4&_TQv;V632{xMhH*nG`F<(>|g#>_&}5vbB|ZjKN{k zFD>|Yi2D=^CwO{(X1{0wUt07imJ*e$Sx!+$8RQ7HffV5w8?xJjhA zVWo!Pl9k{;4c^h%gBl714Cor%zmSy;q8?@e8`0n5@>C*JY0E6#5Ph+B0A~?ZtXlr|2T|6so zOT_gexq$EwuP-hK7u5;_z|?G%VPgt!7S>EX7fTEg9@-`~8WR&K)OEDi5I1^-sumE5 z;X)6HmUHqlq2+?>k^(R$t}EPX zA(Jkq^=Wi`1H`7G>hNA!ox@{=Hq;%$QdgaEv~{Su)qzT}a<-~?%AR;zVpyJzwB}_I zmg-+{cfCamv}Lo`Y@JYgw(5dm0W3_AcAmMska*g^&>zL zHtV~Vq}6MGCOvCk1kEXor;(pC`i{>H+FJqBdhuK*V|7Anlg zatKLAF)(561kC`&AVgRf09b(LFg#ck_*bBXmLW9~_$%1yT|lb<=vRQ^4&d0H0Kdk9 zeF2nL#5UL)(ANmG$LK`q$4Qdje}eR5Bu9@)j2@BlJi-nhm+Cw+&3R0U?})y}B`%Ll zT7Hzi#zlTbBvD2rP)5hE+&6Y*1XR{zz+^oNMAle%tZ2fI0*&x!0H9K3Z8^P_^TdptC+!ups%(<4nj&s81`xh)G8!S z1R$?^B&3kMH2`rgN6|gP*vjxjM9EKs}9sc^Zw>9uM&ppV$)x@6-lvil7>RcIp6aiXb`!>(mL@6hX5G(&+$+3XyMBiKZNBzf>h z5j4X9N&V1A5#+nUk@{heB1nF=tRc2EViXwCJ%EtH=HT%lNbwi{c<`gKz(@5oL~N2U z=uvDIdo--Y7#JfDo+yGc2|^g{Kodn#eHeVvSm>e=uto8ZMdPNCfSGPAT+wKZi?J|8 zu@FTq*rDhF_reqHLJmdK+z&L-=!qQ%NfZr7G!nCmgrJB#437A1*$+ds-BvWs9tfgt zSfOaDUGPJlkV4TE9|S!#5_aewkVE6(cg8>s#lj4Yf*867=o|GKyo`vC>o+CIH3sOGX|a~IC|a_ zx6}*G6FU9?+%pcAC%EjIXP+-5&*Q=I+&k!I)J+tI;JF99S5}&!CM{-v$MA>PdLEwd7eed5_5;W*oyiV%!met|9@_2lr+&5%`92WlZYrX>mlE;QYr|a za51(=WPM7}QC8?!(#V+U~bvR+MbX?+`yly&yBE2#^`xZZ?J`JpP7>%*hwzWu>(Af{4(< zSt)iumtx&Zn73M{`gp+ZsG&DcWd0=I%o1p}_V|4~2+%z(Xz3D+7V@Sxl*|wTqz(|O zMBRg+Ee>){TKv&=A1FCj97{m&F9Hu9M`FXgIaD#+IWz73>rkZSN8_?%5?S^_wf;y>A|CQgy&HNeTN$f#iR zj(&bcd1fwa=tYDP!SAEdo|o{wX==CQhL?tp7tyTDxu6+l8kjD=lPSZ_Sxw}{)*OZP z(XEa^N4el?jk~Cx$NClzMy)NY5?F2hwJ!k^!B|=+q!S@iGuHu%%`m;avb`J3u`qld z_dv|R+aqhwXKS%XO{bySUT)VjiwU*0i~*XLB^3&wUf^}3%?g{iwB1`wKeKErbe8y% zu)Bhy#7#jUGxteuZ!MyCR&a+?KBy+s=<~_IL56%#J_+y~8|3N+; z^z)LB2fsjbIPZ_r)Xvd<AXZaX9DYz7%9t)!+ z!HArs6rkzJQ6B|4_1U5=&1B_fb6BD<+2O>yu-|M$Dse63t*B)e(0*mvb<>6}7+)TS z<7UNOClp%yEMS9BOoU#Eus6lyAY|578(fv15D*$VhKAh3Fopi0Sv({MB#td(1QFnL?YepalhmqlEgu)yvVw*C!9 zsylnud{~-%sN-r?Qc-1eNW9!R<&U73Mg^gc3qTzeeEQi2o<1n(G@x`UqDz`wtCm?k zsLMGj*3SLiOKU$ghnPdqB_8BS<(yQ`eX5*IUP~cMv;QEoJ`aW#EEvgXxpMh2p0TpN za%{q$RYjQN#G$?5%M>-D9*Wv-bN#cDnx5Ej!ZOUX&_#*sw0m65C1dU(Fwe&xSg!0O zo-Mm*`m)t%+B7p#y=?*Ra~Lp5XqHqMJ&5g&om(%92_Y=G!h9)~p7e~u$R7c!mV$*m zvA7IEHoa=rv?V15FRvO!#qKQU%5U zfui&k+F_$bO>;&9L`RC6rpZ_xJX+Lr!en^0qD9R{m<$jdDQfN`?QuV%rkmYKNc;^7 zi5}l=<>y+y7JDz)gH?%?mDvVjZ%|^|%LbDI_s$s4W%H5qDJNA4;Yu&vFQ(Ux-z&~K zT*rC&V{M3d)=hBGL>XOEbZtRASlzd+0Fy(sm|SRTM4`jsd5!XEwq~m{$ffCw^ZW+A zZ8lNg$efi%V;{`&O{iw#m~@YfDt%tc1-s-i_!V12ZGBzCZkYeR1svB!ROT765p{Nk zV#0oxvdpP|LCg&^GYv+3Gb&=)j0|nWrc3vpoHmKp48>OQuII%loR*l87mb)}1V}U| zVz4q?pr6J3S0aM{!JJ~xc>%mcf{V8JQSjIPqOgx*nKp7 z{akK}$m3)7;Z0*RGFQ(q>3JmW5mOMGp)-q%`9Og+Dk>#9?I9GOWF@7}pv;k>eIZg? z!dYOPegZ^yII@c<0~19s|CpfXsS;ofK@$PkN4BXWaU1^~S`0Z9INYt{MHn1*HNfr} zaJpT@J_zpT-I(MNOaFE^pm|Hlhzoi`cwV{mqA+~YjS0uY&5baT?XGz@Rm+BrDwr2K z%!o%R@ehc5-dARQ>^-oZq67=d{e^TVvwNK)xU9F5XLVZ0Km<_)$yr@9*Zr@Rv7E=S zH%lUdcwG<@4+@k=s326i5;@bYEVdJ{?MD!$Y6}^$`+z&oz(0FdB<+;Ibo5pa|F$1O#;`lf`e}kn5uHWMCO_$ z*AW&`0{J3n_O1J3IMb5y%r6j?ZT4HvSkrRF?kiA`ZSE7GX1P+#f$1b~R7q($cj!3Hf;uJaE8XVb8h_afU^X2pe9?AX=f?OOZi^DqA} ze+tK4u-h)1i?gtBq5t@0&-8Sg*|p4^A1yFHt@_O}Rfqn!gfEA{v3u>B?G;!vC~yWs z_+?ETWHPO3vbGtV_zN(=?o+KPR-A=$omlYzEMX-tGDiMVC-0xI<&q`(KT+do()jy! zYMns;7x!QP*|qmWA{@5bcQ)HaVByu)UC-?BkcycQ=<}Ss9z^L!3kMd6q*>M?qO1V! zCzLT%-SW@X;$v4O%1~7u!fXodh5RR{f3DoRY zhxXIYkEET?o_V`wn# zor7-OcalT!M6H>A9|=bkO8>J!a9zMd9*WTGe^xY`8x~Z9+Tovi>DrH5C>-XqTb`u} zEinYb78O_0hSeDE(@W7vevb7tJHgg*ENH2DQf0Js#auV{UWaCzUKmLv?NyiB%yavz z?Yq2SC6|a8J7j|2Iw(h2+{f)%xpaJ|KGYs6-~drSLKxll>9NPj>k~UCf7`gk^}!Hg zKN`mN(H~exJ|F>_03}5DULW7=j=yKG~`Oa0hXl4wJuv=TfUZsOPvjYJOPojq| z%3qz)tX#c00kH!H{~)tRt^$T015)|46J?Ss3DP~4&s0kMc1}N zbQFTT^z-38k=5}4EToy5p5492S)V&9$ECzK>};%<-M708K*d?Vh6vsm>b z*K1nzk^Vt~-B?7_jy4X#Mdg)}ETNf(kwG=Os5uDR0m^EHTqLrs@{$4|?l!?pt>=y5i>eu!M-sk8Zp&P+2pwre>rSHfHvb6*R}k z;YRNBW7NxhoJyJFYh(`9YZ_f0GcKa?A?jjAU|7T>2yE0}y4u-idnkeZVVBB~5>f5I(qLBl|M6I^v;s9kBoP&N;~nTi}(c-a|nqNm>6!b@=_ z)0>8h+<)%9mG;Pq{l~)D+fIAaN(NYa1fv;dgbhliH#mk=eZ0P9_fa(J_$@x%YT}Zz zRl#6y@Or^@3#BS97^wuiWBS-ru8h+;H6D}?8gsuDik4h0x!&M^dnw<^()vR_^b6fs zG1PCLlxtnR)T(>kf!MVeuKkhy=Cftflwhv{x+rS}-txIsCKoibeQo-A)6LGaZsz8H z))s`u1VVQd7&~7<%n4c)^fmmE71Czx)j(Sh2H+}^F$e~rg1qJ8B|hIMvT_DrCtuBe zl!B2Yt5nH~{$sLegweKjFGRhPr{wSO-)H8(Uy;9)zY}sru899%{U7poGSi!EmKNI0r)NzRW)8>@y{Q?Z3l_R|AP3NFOv0~oy5sdz!0ji--O?{%nO<#u zu}7Xy1mT*;JJZchf-J@};*`QfNZFMc=zPnA<>Oi}SV^E=dk+g^*D75WgdT6c;P+MZOo zu|KVSfaGiFO|k+fTd}hvVJ9OZ?wUeeuyIvN;n9~lxiH0+!NVA=2sUuey=P`=LO;#F z#{ytBN@9aq*&2)?L{}065N?k*F3ti8BmAZ%F$wH>=V!7mkm)k3u z@gy!%3yR4#-}89%THcd;hT9IP*Sr2J@TmKfIM@Hf;TG^I$LfsKYFW;ZNUq?RzsWJg zkrwPZnUOz9A&ciI^$IAj?O0jwG=i2Xzo>hQF)N762kXT1eW5$sMwXfR4uABAex3-v z4994Fb^|vXdOZx58V-SGu^SPA`u!k(5c{He|FywVHT&RszbOi% z$v!EbTU`2cWJK;w{k%02-7?_6a9U`Hox>VbxI-xvF|%E6jSR+>3sB%G6D3b*=1Zw{ zX|n+aYCG?1;lwU+PQI@p5JraTG zhAA>u=Yl!UmpS(q;}2DvYZKNvVe=KZ)LW=^)a*I2mZ}tV!;bnDM%Y!2d3Y18)zP%c z_I5|4XIh~2J{Zy%j@+Bg8EJ9y=nTL5qp8N`eKjDAHfD2Kc3E!rz0W)LsW@Z$uG-e* zTo7wU%{}e+dT6?jt)o)OlsG`r*@}| z%>i=LGS`0|9RPh`#(`<`U!ejX@Uu|a^%cK^qT?|JJ~W1|S(cN6rB$Fk2TT&U{e^E0 zfP4d2^zo>1kVn}1qyLgOm^NPmeZw+uOTzA0u_d`wm^%t2o+`yxViWT1FKt{JXppcv zt85(^ZR-fEII=;WAJQJ+8S%q*t0Bdoe0Bcx{OOK@-wV`N{IB5uYOB2c@8zgJ)%mmYrzd*N&rbE~Jsr@>{rm-i^JNKyJXLz)SM0jV#!&`jOtj>9w8rnZ5@| za42qRX3c?}e=CFRXtqn*JD%D%7QUbf7poUiEct36{{8sXe_XwKfBE*+yPJzQ&&iGd zswP8AzN!jS;TY5Wy%_T@zUI|wU>|cIRO)byYm#v$%A4LwI*pBa0S`L|5Br$Q!X&B{G+}B^uB+Ts&<&>HX-W6P zx-9b#L>A=3s>qWM-2y>>r2Gt8clG>9L+sWD!nmEtjSbC;*^l~Z4B*ma)$M=>3qIQ$ zfOfNHNZC|#_cY3USpCTJJ@!Vww$BDIk>-+%o?Opnrl)xn1BHx79uWaDYpz}LL|(`{ zp7yFVTo}t9`K^|LfA{>idKWqqRl&&nhKW_Veomh5v%oc@DeFrk856NMD5$pcJWc2o z#3C9c4sH@vc2dJ_1Aw8g-e0}Y_vTMGH&?cBMGtZf#M^P9SW*=+cmHxpQm#O~w_FW~ z0hoI##UOr8!_pK|u%y3s9L{MWWyU&%Hl^htE>^D#3Q&5Es=r`sOqte*;;fD3Uo)GO zSjYuk@QjyRLQV4yK>E6jCdhHU3eH>OEuv(ZNooqGKOyD1kkx8Uo_4?Ynbn8gqbh94 zH%wMBgypS?CbD%P4WqG*tM4g?3`t{Spk;5Z8+>nYL!bQJ?!)P}-jmaN@87BS@3r^u zjrVWm|E^Adf7-ouL%DvL>Jw6SHtOchH8EmIDd7@=HC37NLxw4LFMp7}B{Z_9=nD;J z392-qRk@ahz8gT=U^;iK;7cf83#Ix)QR+zzDiE$6m$id}+FP2BNN{>#-?2f{P#%+( z)b4LwOCM8Fo*OakTS zPjg}fN>-G6ij+w$mXRgT*sg3Dma0(dd{Cj%IySCvdHx;PA0yZG5`7#t5fffx& zk};Tb%eIGYG}emvEcW@9ZLbHo*#IZ&TGy~pJUI6_J~znY9*Mv0!P_FK?J!UKocY;U zUKYWrBKIwhLyhN7zaiw976y`=zoM;~|gwCIE3I4|Zerv)^(0>HmYeQro zkcI`<+rODJ7b&sxrvOA#GJPQa=*S>Bl&`PtGg#|CWqBsIwup=?0aU5Z%LM`F`XUz_ z%$qGGSus)9oAuydm$cN^?V6S3pIw|2yw(9(6Kcr2A(u<6Yg?D%ZWJe?N>v+X3*nVa zm88m5Suonw_vEf~>GY2B4EVAwj15f`ZANeXQs)+UlewqLhEt^-DFp#+!A>P_Wi?zs z_2@hASxhunHodjpQz*mi;!Qdx?n8l%Zh&=#`tzI#Q=1ftrvF$6SR*tX=WctCYvJIm zZr5(nB*j2_Oh4}4;%n?kt_h;Sd)2YIaV7P=*?WagDB0*=^aX(G)uwZj<}*Ff{j^@x zJ{9lbb6WO_iY^Bs4JTs{+6Y^;^s$17lAeZ!4zjue8V$ZWZO_2{xz-yqS=|a%HAU%#$?R(DJ}Caoz0b4{s2_D(2WRK4PXK6>U5BQ0o2sm6mTd|92x*LW``*)m zV0z1iI{&KaOqam3UxYj^e@M1Lq7*CDSfCxVG?mk3RpwRMkUc>cGR<=B=js-9M(*i= zxGM>nuMf*Kz68@XS0mFLB+@hsChN0HT?1Z$ojh!9^lt6UbY)Ts6UnLAdd$;Y_FQ(x zoemb;zgMp^s!E>xA(P3i-i^q#x~?}HD;4D8`)l%&t6SIMzh7C?%Ww5gqC~5j8G{rS z5E^XfW>*`Ceaxi_C(PQt8Ee^4h!^JulYU9>6~pB&^m6r|S)ctO#ssxg{izpS+l|lL zykyY9pfd;DYiq}eXsQT1y_FTYrx?Hwlx9wDc>bKc5~{-LK4t4OgZ2r?Pjguei&|^9 z4L8Ng_7WklFyU>$5p~2qmR}#8k)w}GrT-(!r8=6E%T1o~gqK=U5ldDSh%F0dUI@bE zOJ6*B6r>1#AI)4^cupm`*K#wv(;N|cL%O1e-sx)Q9fSzECUV2Qu3&IULbkCe%NV3+ zhjaz9gQM;hi>BTW4Ur_6Jg-Lwf?h6(#1fNdrWz(z9^lG76a=~0+OKO0>>(i(STuh+ zPubnMO6k)xm;z$TfCHzoPO8Y!)1x`Lw(`-Lzi#(j5aLo6ZX%GR{p5(8$O2}RP(d+u%l7rmJ^KbK;YW4gFCH^6C%Q zW?H8|(WV6{&yX2==JqrwcL(g)^v9}TS}utabllTz;YR;XUec1Hi~|%Xltb(oq848k z&^!lT@r0BEF{!TU;8hIxL3kMIpXEcR&p=bEd?+aeC zAT^3&K^nz}I`=jfWM8n0@qJtnIHMmhyeP~pGPxiI*&kvR-@d;bwTN%uU)nY3f6C>~ z^kIr&LisS$A1SETa_Z}rj9RmDlL!}_J>8LUBc;nA)K?$ppF*xYbkA?O2nLokZ?S$F z(#w-N{Th_s>owJclx(W3c)0LAhtmiXDN;zSt`i9VovH^I}VJgAaWB{_-GR9SWM+<<PF@XvxMsy2PuPIM zU~E*6$FBEBru0iiCL8`bilf)BSbpgG>DwqAzoT4m(TD}c;72YDEdtp3j6u*>2}kvQl!Avkk?ySK*uJ4p_q{y}^49y6l0;@1R!sEP1!HHvrRn_BOA)o< zq_NZinPN~~1BegJ2FRyD$}!v~9B|v->-wGMZeKTkAJQIk6S;fKwgmbVE@h!{vuNTH zD-+m+^GdN|j-UQqHg|~8!DQdFo;oc3Tl=0ZyLQ(6?EJxDf`bxRGsn-01D#uq?IwTv zy^YPmiOEm-oTGUkC7yRArx{GC*e{_%-_>}e*H3E%ls=0ptgKhiR6>+e0Vf{jQXV?f zbi>7GLk)&yOe?bNEkAOR_V0SjqzTV91xWLoal$wsbEuTBl^&$9Ug)L|y7O+ew|}!$ zf6He0^;w?Q`~Uekj+;iguz`p@no-Vr;zpU~h9--^VqCrlpBwoE(%)yP_eb%~0|2=| zM!&W*PR83`4yoxfMD#1$nS4+bLu%!xGv&0P8&$+;p9>5t%k*Sjdu4XzvGN zQdqZLW1aUB+W%pi0AH<3)A&PUZ^n}n%fTY)lmpFih>iNgIXE#87JNoBe#^*G{jrc$ zUiXp50NEf+(^M2EQRAXw6e}xp zOb~{t=ggGCp0+`l8(Q3I9nnzaIb@wES9GFa87=u8dY3kI>t}Nf&h>acKSmNF3!+L| zm^-h356J)#2K!dY-CUy>vUXePi8gXcsM)EXuD;A|vm+Cf{sP>5(5f(-0k+&RvEz;5 z%HQnCm2q_N<`T2kGUhX5pW#M;eNdK+;}IA$jxp>R^NwTowMpd6h4%$2OY0z2FaqKj zjyxRZjN^g%GLAjDGmcmujU$dz;~4Vr;b`13>*mAd80?n#zFr>|STT_vm7O@E{fRM0{B`fqw7T%Ssr;V$`B@U^m&cZU+_y!4@Knl0A zDvZy|R)0xkp>^)QMA&*j)sK}{K?<*dFt+77I81lH(CFIb(jSc+T$3rgVtebLYy&Rt zx_eMgm9Zv*t9+=t3=Z$uP{ji2$W?u4q+P|P21-KnJPSJCqxM?JlwDPejH~taXrq7E z%NTs+RQ92!xWsuII&A-n3B^G!X;s|O$1t=%?F%Czr!QlsPLKZ6yL$fL4X1c&Sy8|7 zt;qd%NJ6trwd-M6vLBYwHAqll7Z%gFMnbzI5W0a<#T?rav++(8MK%(XPX&>}0yrZy z%j|dRdI90t7R+3L5ZX=7mM`AfvVRn{)eMma*wQCMrLXE?8IfgJ`-kI=$s@cd%YrYe zGK{igdYSoi7(C0yR<>sZa*bBqfsD<}1}EM%i?C3;i@Qr0YoP$-0sJvgw*@9FlL|U6 z0zGav=|j`P@W}*GHzGj-=DDv}o637}PmA>8syBbEDU#LC`$DPxndsl?0@~tZ4nx4< zwd)=GDZOA_1~yX$yY0yeqC}^cD`dVSYJiLF;Z#FQSrF>vRGXY?Uxrid5}j(AlV)5& zQJn_8v1O>jR=Tl%i`lh!D?LfaEvv=yd1w#1gbk( z@igY?b_J%os%UAt;BP4W^R?c$o(fxCgDawixsqWq6IN^#k;}72T;9>$(*fPxjuCDt zVi6`({H{56OW`?Fxki?{aCJl~d_8IhEL>~c*+*DzoPnBvc(sf^#ppu&fNRa}LHhk0RWMo@5c_ zoQ-iGdh$gSnTV1=8d63C*e}|B=rPSJ<4N3gB}Qz5Z{LP}#^Ml0+@II=di7N^p2V2D zS~T2;$H(*eF$R9r3PV*cXa7o=NMsu8rPgySR0(2LZHn@pryD!pQEo|`--EV@v2}`G z?2>9{)JZnFn4WmQ0Kr}@|AcRbr-fs&C=hQI&q6;;Y3u3~fA0LS>0>FP{Z?udPbVdzv$#~uv%@JD(e|DNSQl*rjXPR;{OOnZi zW)bD`NK=Xdq8JQMdqrP5pgHCb)JxS(}0&qtvRGssd%htN`WTAVj$Y_r4Q1 zm8*!pa~TEC;%}oZVC@c}5+)GslU$$AQr6j1!Wm|6HsURcyB)d5o9xZ24DAKQ^)6?z zCDg`oIqGj6#W&7aW7uA@O-$t$9#t(Kr-2v)i6Arm|E#gcdiWM>i2*D9w?@MP{o16% zq%314WUw$-wjZNsHu6p*=MRE>tI*Nf@19vcC-k(D9jZ`$6^x~rACoZ$zNmycBa2fT zBG?aDE{V_zg6rOZR-}UI4Td^4h1?J&Xs*^0dfg>7rwK0uP2~!j-0D}sX9v?eBgv@= zZ}yL^nc`Ef3t6rDgv+()gF_u(*uY$f6l5CSCg|fwjOqqCDhtHwF0h4s=fP)S*@9pZn3FfH*AM zTY!mFdT|{WSC`7y=JC#$0@cCAXHu;EdRz4k)$BkJF0>w=Ry0FKfRpb9`QrK#!SdC* zoZU0<>>^LEX^ovEZwILM-!cV^rZjdww8rZt-qS7A%`@?kjS!Oo#oAzG7Y4vGFW|x^ zRDduEMsa<#s=N6bj@>nCMDPlQyPuIID!RcgdL510aKy;(cQyXpuuY!PGCr2$=zGs< z?s;GJou(q4(M+S=)psw3A_d${ZG>XoSh6Zy*CQ>cjk==*5rAzLEKTd)GiXY=7p$XN1q$-2Sg1NDIcYfl{t8#@?lT0K&r(JbQBxrrswD_rq`X!u6ZKoBh`7cJR9o zn&TAufru6%<_$lt+cj_da3$00oF#M9pXUQA#)OHo7^tvyB~!1HGS0bRf3?vQaAeMI zo6wSp!GL9(4J%}6n(8jWJ*D^L1gwQv{NRj~QUcupS{Q_RAqe}(6;wjOy@jKp7;NqY z6#Yq!3AXS}7*VA`pq`g4I%`ILlSx?x$SGENq_)@{lFHOQ(kPzhO3g^vN^fd z0*4>EfF~&k)aYj^sYJ=c(MS4C_kGv~Ur4!J5>+LdDdq*6S#f9x`MU_JDH@bZ6UDB- zw5Ax@m8XM?hqIx=T*-DUH>_QXq!xwXUiWb{5=xN4F z8xxHYwc)wy6WCCJ-NOo&NTJH2O5mg@Srqb?3749nxxr0H2qZQt08hq7utMmDB*~hJ z6{}OZH#7wt`LojihO8O(H0V-G*t~>eS!jX8dAxoQuBPePaJOT-g*@wJJY4TE!rJ%Ux@65^=cNY7M9B+g>l$bTHyd3E zMC|644;!X6&0eum)xnv_&op5)(u=X&0JDQ*7=%iVv7%csz(Z8h@FmTChZaA^MTh<1 z-pVx2pVw;LR=->)Utn^N0`nF=5ao8*0F<)b+8&!f1Ii!*nJdCE1yw?)g*0P0+_vQRHFT6oa3;b7dA)+7#CuToB7g18Z`Js zL0#!(X_ZINMWz|LhkS0nO}-bmr_zJ;o-K`~#{a1aKMHEidmHeP`8I&hT_9@|U?bUk z^&(9=TOAMU+o|G07|NYl`IT@wR}vA?F*c?HB55HWEbi$RHM+REgqN*exPt#-xqY;g z*p3C$#WDjxLI9n0-Rp-I_pz@tX0P3Xxhv80yH&?U*3F9Yq1G8Vap+ ze2Jcf+PV-&QY|8ZWp0aHbZtA&f}(7y3BHzb5seF)+}_h7RZ!r7mK-~_ouSm`3s~a# zIPQwp$IL}r&h!_3QvWUPnzm)sQz!c>=6ZnDn-B-ID&>ZjJOK+JUwYoPjq6fjV=3aU zD2O5;ESbYHcd}brjl}!*MtazVN8{~5+$p(z!cvGv9y1fZ!4MMgq!%pma&ELzN1+Yw z($<7my9I|msjCH#tf4?c7c5}H@DOx`O}-b-ioLg_E=xR!<*sk4?kVBl);QXge9 z)JNPw1aN2iuHUmP)Bg=o>;cnvVD)?@FuR{CYnn94H798y^E_5i(zt%wUOj|?6n9KP zFpy393AJbN$BGG7&=MSfg|(qxg_sdBJWs%2=MA-PKDx9yrw*h^$1y1shvX5 zs;(KIFwzU@2)a4$joAQG`%A+#fuVW9?zpVt_Q9kvR3%Cnsx-R#ny*}ER~xDfp*KYu zuMJ~JF)b_AthJIrK*IyNxZT-5hQx`<7_sbXd+%YBZ%Cs^uKnH^nOtsoSuz{u>~03cr=*ef z_55LI0yt1Gu9~II4=rj7!Z$g#LV&wHGdXzHaWHxZ5{T3!>|?@M zY9{~Xk(*JEV+nw`niLIuwlA8>e00Gw^0$f=TcYJcn-vlp!;?s%cxrN2jJGaE6BsL| zfSc~sM=cvewX41K>o$;PaYERNXq0o?=u4~^EZUG^W8G?cT)cZ3uSbmtVqu&R|5YRz-o zz8CJf!3Odj&G;BB@>ht;?xi>*?_{a}^9nLZM&hL&FC|m&WC`zol0(AfA8bbaBN_Y5NkxM~jVRmn*(xH?syQ~SSV48TB&z*YJx1~<9DCyRxbS;B#+Vmz3^%}P zK5daEUP2zLq>f)X2tscl(dpA@wtJlEuH=RjMyX>%b4_*rTMHL(f&OQN9?J_gCl^F< zvC5ba!fswb(luMR4y?@w{sIlNHA+?X>G&y zN|IMaE|ux~)V?!ul#{!P#x^|^vzFZjffQ1cwj2fVqFJNOIGCXDl<_Ug?v03oZbL~? zZF9YDaH^PCete8^H^!zOi>X?bD*YI+QZ^R?>F~pyt?f_@n{a#m(j6}uS4_qqP>*XN zD`D{7R{1)7NA>D$O#wGOa~tZ!z_>{+^*))30ai2c{Fh5s6nJ)vZ8znL`b;Vro-};v z49~7RhI34IoQcq-}L_3ab)-M_(_fZf^^qHA-zp6p*ol_sT2Z z0X>yV@;a`Kqz932kZ5AkfGi0YNmiv8KZGPy)2bLczsHvDC5_atS{cyhySUJzSOVWg zcpVLMrb4jHE5wYh zvEDh;vZToxc>yJwZe^@IR%9(zNj5ZDbKy#43Rg~hAb%p4y`oBt*?MGa+GEq3)}UwI z+!e2V-Lm%WP?;BKr`CSI?z(5l0BjAav%s9-*)Y=fH_IiL8|Sm+!sE-#yby%T8`90( zG|ODedy-0W&ma&<%c+oApkg}ac&mq`yhLvDLTVY$2IlD?f?G!ck#{Ej_Y^&PV-(CS$faY;UaH}oM=F&p_N3Kc1LxB z4I3}-?|C|6GvM*&CRo*1ZvxS~KzPbTsV{dCmeWoE+9`R(u_PbBhW0F*1s8RF$NB(< zyg>S1Swp{KB^C{Xq7SlM6U8@GR#L%arLt`hoj&Cu3^Uh^eZ-3p$7J2(O=b&eTRgD` zHmEk(0F^r9Ha&No*A>lUA6a6Gt>js?;=QG@K50Mvy0v$upvZUlD7{Yjhq8$MTGW4; zV!%yjdwP{^25z!LUHIF z7TZ>}l+elBmEr_qY;Kln7P>$KC0`N|4qR-`u93}a?Q@+A2w(jT)^Mi+((w_%86+a( zzg0(9a$HS#vSy{hHUW`$J|?HGi5c%a=411zMqAnB9V_lQyFWJ>U$c8&u4g7$`5Xku z^IyV$2CWUKR9&R0$=Squsir!^+-vcA2B|$~q~huChpb|wM4mz8#Xj!B$8P_=go3?s zZG+6SEsKWv(Fj?drJ~=O(PP0dch+!ITguLUz!=v9)UCMVHj$GQl@R@h7A2a|k}6gX znd6%BS^bWt9{wFMe**s#O9>qy8K(9unf=a)d7=SjDGRqTw%t&e{@Up-u$?FB z)-XYQ+scaEn*!rTPr2dwbMi{4s$g|fpw?%Ga5z$HA+NC3yn1f+lOW)cCd{^(G6?&j zu_5@eD0^w!COV{LJdV9Ir$q_sRy=aLS1c*?eh`OZ=jGBS^fdk8T)BtBO17&Zr@4@i zfv;U(7IL1lyK|M&r)MyQm~7qVI;kQ@Pmkv0ns0bUiwp~bxvFUPxv)uG7H%RC)b`|v zoX7%Zlu$u3W($iM3bRG6U_fX4RAi4lxcNIzM~N=S_`Jtbee=@Q%i38rpqBjc#j82_ z)~GUuEhAOa>(@99PkhRajk>xASO{YD`7jS?>H51qho54@!uEt=%>*%OZ<&gh?^)g( zZrGBf@ZpyImzVknsW`MUr6r+AOp+?2A-@q?f9K@otE=~~UR>O~dik9E(R_jX!tSP@-9c*euVqnxD^) z=a7?eJU_OASZ0GIgFp>$swd<3Z)zgmoRDv9a%)&(<2&)1F3hrSVcGG8A=(0Vt7_p- z)`V@Y_?@Zcc(5(Ho=)@;dzaIyf$!fOJ`cQ&BW7V*e3>~Uaxv~xG9^Y^bV zUcP-b-=v3-C??WeauF`k(CVXYU$MAjfABhanfgm%naS;ADRY_0Rd)~O5gYMK3`U;O zncm^U)tYiWMx!a$c0=lwIcdIJM`siKuM??Iuh zwcSaD9!Ab%S|wRJPl?)YGA?fGCPqtXiX<|k|6uXAU^JXx(B=*YVZg!EzZ_*{27|J` zXMeI}!UTKx%KXt0tZqPJ|A|)~q1qonbGb?sk0ZG7XoK1ca(VsY`m*U4*)x7RB7i!m zAoux2`j{!K=3!*jWKHvns$9S13W_ks3n)8pdN~-_DcE=;2-h1OMARC_i$7gl5nAcp zD@~im10{ocjn|mtJbH!Zy#v-~Aw0(F9yWZw;l~=Vywh?IKUNr~nb~MRR@i^_2}1D? zXNmqKeL zLxr20@y};OuzNEnZr>-rHD*QV_L2SWd35{ox>cd=_-brLi4=k*Lrrtzj^uCgWRL$% zv@E5#)c;%247cyyLzwxR+aEu>#%p$>^)bA2_L52sM0-(TR#p~ND1B~Cf61|NavqbD zGRcGfy$8>M$tvNCD>%;^JdQUm2PQP72X+DiR>xbk%=7Mi{P|zO-^d%TAT(#FA3Z$G zm1zqLlG~M5I(hyVnx@5b*(gi6+ltsUw{1sB)PAi z{rXAp@IP(d$!JT`C-p8KEY6INEpEgRLv0EzWnE)E63U4l_OQ-{Gq@ioSU?EC}^s=-Y!R`uuNwQ=MsM*ZVhrllL_g zgf3q5!ug>pCanql!k8rV^4;}c-dz0Q)f?D1jBW{7B+Ipt%J+$TjK8&g_iOOjM6As}&(U~C)UaT2aByAJjt&bh#h47tV|?DqEc*T+@hLR<*My%b6;peyjdedMW*ubk zotbs)OXl;gQbchSuJct>LdaH8s_9?gTA2(sThOShHj<5_p&PH8RzH3}T)m-onJFm0 zauD$Z2;-td-7D?nl$G9SFxWo?gD(`(nzXyqCq8A|1_F zS`jqnZ|x+~er@FQx_j4f_gek-`tD#@B6dpa$SWB3gE+ri>k58eS8snVIZ&0svCZIW z_NvNou>OH4KZ54i7mOi8A6!jwC>X80yY$#p#Li}KN>a$bLaER=B4VFL<2^R1$})W_ zQuuAKOksuoaC>agw4ph+fhVcF7x%PC1I?nN(i{*Ghst6$6CLVzAh-^Ey83@Ccxh@s zn+nmMt{rxjh}tQs$%rNy<}03EOc0rrG_$QOS$WS`SektDR+Bdos<7}@ir9FPfWX9g%zuCMsOfLV0A|R{ra1Cb)8i6*C1e1iC*J~+s-+8 z>4o3}2Rkf5x#Z>2D{ae_r3|*<2w$yj19FXH86T)559SoK3)`AE7-k9IahwE} za7HqI%g9ne>+x_diLIerFr`I)Q@t8LRAZN7!6HNt>x_)(Y1{M=nagaO8JX@{Hjo@? z!N6sRaPl?2dF7zuF_;1}HQ}L|6e~Ej0sn~fgN<0~vUs=ofX61jXHlrtzp zZ-cGpw37*`WqBVa8C&k1>@WC<1hG-Pu?$`uBOV3OG)lS#Oo0FFd%OivM! z5_pNM8BJ~XX)PAzIZNk+ywI0%$rD;eH8a$2r!1|sTy1hh*Nn&_Wmq16NfHmB;X#qt z3pfGSGhFbppv4x27khS^SYdyH$Q_1RVYf!ez6y)`WOPG2z_aTS{UuxU310;fZ})Pk zF-`5DVGc6vHu4knk5EMz@Z3m<++IBwr^d>;*5JyDX4!VeK5FvJ?^q18ume+CO+N%y zo*`1_1uL;`&k?ReNXW=If=44kL*s&~#Ht5}Z%58OdU8HeSI!6O%XvU^ zy#0D}K3aFqN9xb{7#%tf>Ct(>f$h_$^YJ=$K31>J$LQAii2XVrrDNx#^z1yKYv-PQ zI}gD1{d#xq)4g+S|IWQSc<$B1bDurfqmSpIojmtE`1n24T)U{)8A+u$E{%C7mnjy1s`7Xh-wWZ_%yjoKPSmMJUvV5uo#`Fr8C_&n z&xcSawDCdmu??>3`dI9|m8;MoB+weDPvL*CKJ!>Y<0jW>e|%@mk4?N7Ki^UsQ_YX8 zkX=TfF{n1=d#%S%IB`81c=jGNk2~W&f>umk)viLy=(gJu45aA%Tfj;mt2l-o? zk8~-y_xwH%gf3)N+U!{Td&G7@1@ux{@7n_#r$?wJ%~VzhJ!rw$L)5g z&bZRky=7aqyNXzTV%J)yj_d1TLOm+sePhgiLSHRN;aAKQSEkmk=OwAHc@j&pI5y@r z5G^LS!#;~G>QE2G$|}d{LWu#rTBZ~>WF{M0+_Kb3RGKD5MZ^3CAlXU%L}*~1=I`9x?BP==`4af3Z@@Z=L9!}&Ui0MYn)%Kj_4YQ+8PBm z6yep1;u-}v5b+h$4BU>`5U`zPMTLa5N4y?}qZu9Q)g3Jwn%O9iuy-V&jpiQloRhbP z;f+Kds%2m$v#N9@Q1|AXIACGcfQXf>Ey$Xij$XE5iSbO>ygHsijJ3Frv9?F5Lpb)d zB*t3T_cQ|z7LkZt-6t+Jfn#jJiAb<9WhrocI#Z*4)wVrxW#$@^eB=UUK{Sm9<;RNK zSiUZf`HwSujY9lGtl>7e{^II#q&2~Nv&Wv`J@Lg_6fV)C-~}*|m-Vu~8zcTiYxMgo zPdih4tyyIq#}Umo`~xkeNm;0wZ`~X;-YI1Ahk?*MqCy7n2Z|lFN0Yk0x7^QIRnZy~ zKWiM)mX}GNgENj$A!LccFlT6rALU3SE{xdE%Pzd^vr)3+zOhy{e>f;;#lWz;q>?q; z(9v?19jPA<@n$aKV$9a?rCJSWj*!|kXYKK9JIApUIA&eS%Qk4 zja7b#R}H{f&@{2X!?XKD+p=Huhug~0g7V04j>CP?I1aPObXa1nb!F5kg)A{=R_qz8 zT!kz#*19qd<^!a*?ZAvmbsGh8$Pxo*H45UeCC0a|i~=37#6;)I?ZS-8pBrtF3R_}G z66IE|$Ir_W)U{|26uo)T4QzuCpOp$eau|T7F!*PPFQ*Hpl2+nYbf&vKc0s z*D1f_YJ6q{729t{T@8XYHuoaz&A0aov2I2M84&^P?y@TLsx-W6^idj8XZvrs+j-{n zrO_a@e)V)Xudz9DT9&L3&&mJ)z_}4n8QDQb^80$zuW)u z&!?v+C;$29w|~63di7uY^zZ*Es?9C_ufL!CCwui@kw8vQfB(NmPPCyHa8b@=F~cK$ z4iS)}z}f9S>|WQbzi!VmPC+1`BxrK~i7xJb>fc~j{=WJBB--D2cgyGELKy)Z%f5`X zG2=@;=4JE8l$J3mkDr}o`KO3g!xHY1P^Un(FE~&QPr6|mfcUd9bB%~n!}zmEps`VE z;Np%({V7UqcuY}hQ%Z}4(})^^Ep|&7H&L2%(|hJI=oI^y^8%3l znk7=C`xvfvVALWLxWl4NENNA^RP{TW@syWa7ZHZnOY0tkmD%fI5mVJ?PH3JNk|t}W zyqTYkIBiqH-W9jasudTg6aZu`C1zVv$dZ=P7__@YAFJ#q^+zHhqR3VjTWHTB&2+qd z;NC@uH!aC5wXHq>^@-Mh6LZ@&kAY@mr6gH=+<@jzOMh04PdxjYtYuZG`O9%WrBafo z@Ftp+tbSdRj8RpRr%%X+i>iz*QQ)a8zy5l}HH=qQ_AF}PD=6@9r=>(!ZY7{FYkbQk z$D{PM6~ns5ql$ffv2s8Y&{B4(OUm#8zO;uJ9tt6XYhy~t-(cCZwh-NF-`yF z9v%FG5Fe;*aKTnwmBpAH!55myX-Xx@f^4d+qh@fXVEvVc6L~b@zouU(l&}4uMncircFBUYbcfpH&ITSfdCa4Jd zPMckH;c+7G1-5EqWX128XkjIZFd6-UCbvwahwMkB&UOXwpheRur4^XP?B?F<| z=e522(r)3q+p4#-!6^GkH+ja?{PthL)@JV1d9g^(-@m$e`S#U(lOED!a%2;&<8TMN zMsw#duhUzxl@+*Yvh|rkISoijTpv90%4}{}qWlEi@D6P) zTP(tQHkwe^@}8uU+(XF0oz~sJtFNpz#XHPpA;Vjp$W30z4Of<}n09Ul_u%9kx*CCw z@VGNNz}>{3Or^3o3-_qo_RF=rpOtb}DK_K5*te(!-?A+P0!A;{ywK47N(n|@_+1Md z@Bi@fTQy`i^2F+MRwm~O%h%@y`u@b|fwyNP1utb0-;U_mzJXjYky&PX#mFUj!Sb~O zIlAlh1Ft_KL%5T{rSRMh)S(CC3%vbP%!hKarC zZt&~%U;kv=G~8!BDOfp>0@7}^Ze9WX+;cS2s$4Ts@&qlrYB=D~Ctc)niCRUyC|bw2q>4gNC^MoQtFM#bZzy8w3)5 z*&&cL-SC0@=+A;g4y$A!SM~nYGSz)GGL_JbU49d#m6>OoirKaK(d*AVpoaAI2>-J~ z9wKDg?R4d7aS}VemiBQ?B%X$@TRc@T642=mNoA0N;Y4 zzb>N`iNfJ`6$#y_#^2sMgp1TR1oi62nI{H=)u~Ir!C<2WBR6ZF++KOk6Itlrg!fal z5og4EQFvySbv4Ft;JsrxmKDsd%`U;;s8TZFgN=Xz7r@F@v#}J|*x{=KVJCiJC`Wa0 zVSD=K2^uCH9@pik*YGgE`hcdD9VKACwv6cm$S=V+>=)Z0DGL9jX=C%yEu@IEEL;wx4@oj*R1= zEaq{!%VUxN;t~L&vy;aZ(+uV$@8%gtKO>rZvU^@c=PR zE@C=l0QGhE@8ar`Bn5-aVVWsy3f-z_W41d8)kuq0rlV!aHu*pw+{uae#XIpH#38V* zHdM^?4F)%!$5n$n&}JfYV(u!kRl2wr z?T%rr#*X0j%3g3!{P2xCLzDvr?JLOMtRXb3RiUB=E!@@MG<`;{&t0$2q?0cg+3=!} zh4JS2%Lu}R$P=w6vI143!o(+Bv(lglpkFBo6_{u=WGk?Np|E`GD_*cQz2mYf5OOKW zk@?e+=9OC+F7ItE({;~(*iF@OW)2%7jLfODhPfH^`QHKOj?0$Ie`JaAPUV`{eiuwJ zYhD5Wji%&ycMX5qS8g;_4lsk$CX@UWvSV0o9J0{#IM!J=K^3)}zapZ4b0BZQ%#r9iG6b}VG9`!3aDqjAAt>hVFqW>@rNMHew)xe z&0ffjCdb|F*c!)i^uorVD9q!-x2#+a%G!*s8BIwhSLEdTi#MkoQ$_Z8Eoh!^=i~=L zzQ1_WfX9^y1WP>+ItT$*h1{@m&8jZ8`MZB3&*smb%%5nO@2&N_eudmUn?HRzN2@zs z$U8RQC37LA@189NcIYKovs5EY)|Dp$L93q4JzJ2zDotTdrF9)Vif$zAoYJ3eZmvj9 z3%X$?E6@!Pn!@ljU@DBJERQal zSyC0e+#)d?;nD*+5L>-!I5P`qX2>coW!N*-S{3*8X(OaD7R{}+#x-7=MdY=C07}vijfVy)soHt?e>zsE5*>8hgCMT#0|AG zxUwgK{v@yf4FFvf@?HTNN|H&M`j^wXh)}zyxfYh_a8L-Q>!V}}JXrH%j5`F7s#@Ux zQlNmqHeC<7Bdlp7Wp^yoI6vRWA9P ze-swUXXKr{)(d-(qkXN}#36?Y0CrC^nz-W#pC%fFQYJ8LN8dGB@YPV(++kPseu9EaEXs9@a>a`gXkeO;&i>~;44aeY1WvWM!!r&+fD*&lLf%^4lOO%WDYJ>6%K zViBf5z}+~1W)=TAc_&L#%EbJID=>eCL;zb z&AtR8L=TDUmFze4Bi~dTBC5@T6?)-{Z>p@Mg2_s&Mh&d(-phnW!7W6Xl&;Vre^rM zkba$%`u*ESzGtA1CW+PWx*3x<}oSh29D^>G6pZ|X=#Y&-X$fsS#N^*?WK z=A+cgkE@eEVyENl<99T$Yz8aUsEw8GBbKvIE&TX8_zx9F7G0Wl-#Yp+E%;h;Aw)HV zNyz6zB$~lrj$}Y#vpI?FLk*EYP>$R}x@Ezd05~SH^N&q-cDidstmkJ>b0WYeJYTco zy5eOxRnWf-))rl>ObCT200Wds%Vc+mfFW$g3e@`Hr}JkJQO`DcxjiF)l+Jt61Wt?L zDEW&!$}><5ci&BAQf-*<@@H4l{YZy2cYDoOGNWPz`g4A}I!}siUdr<_Q|G_TZ&$NK z3RO~32Ks9cCdiCXr$ZsjNyeEd|B}#sB?Ph4vXVsl8h4#}R9 zyl@l1FgVWCoVD$Df0 z69iy941QoYfuIEP>cz`Hy$Y?Cid3!XEj!aMcnd^>b#ekB~JKxo>gIay+91 zHaLt8F-$VHqtp;{3^T?krnnD7+|ZBa@FEGpTjNFcj7)XeGw}LqqFPr@3Uj)fH`hURnC$b73mCT-sCp@Vg|KR-+KnJu^qRp zDO|{sW?saMd!OVoRhaIL;*aaNy5U0a)z)+f?n)&<^MuWj;*Dtkl;xS+YBb(O)^IFL zD9}Yuh4*L(JJM8A6sV>8*p5z(1 z$XHQsAUUYOGP6`j^}fnj;UIU>gCQ9;qY9(dVF5QhGM8}oHJcm+t%S{z^;0IWte;e& zCEq`#O4(`m%BvQ4H0W8N+36t1+AyU8J*ygv{!=rf+PrGZ3WN0ylvy=s{C5a^tZ<~5D@&;Oe5?oAN%|2}Iifh!^_ z@Kl;0Noek>sdm{y;9jr|T40wxgx;aP)x$-q3Px2V7QBZ?{q6z}j-Oq;yk|Do4gT#W zv-hXVfZ?mjFT6I0_31B|zS{*u7xkXA2qzTo($mhza1T5Gq(9%qocBvfY^h%vS0(6R zw`|*Sggsy11%V&j&~}&SoN9S?fSJFq$&%F$sMr#UQNA|YrxGbEGH9NU?6Ql=8nXdd z$?$Y@R_IH|(m653a57dh2i%npb&vIIX1B~;>?fZIDP|=rHe5jJGYDyIX{%#(ox%+{ zDc6h_B%=$KDYBtuVrsI1S|y9qhm_DP&U}5b1!VY>^0s<~6;ug>V#yzD_wy9&`C&iDc*WJD(~-e>CXZ`L_=A z>VB?yLO-2^)3=y-AqOA~v6pLj$STvD+Pal(Yx|a$^8}ASBDVtyzjpC`!2lujt|}LnGwiaKXzXB;j0|J#GTi7ciJ=9m|yh1g*W3lnB(yW zti0p@QZ1Yf&H{#Jl=W9YIKDOlDOAZJ)}CO+RQIf&k6n>{RG%GQ$@(#Pz5Sqp=|5`p zWAfks^NW9)#$$|RO^Z^UCq)YXZ&v3%GBw}QP4)oac=F`Qli&Qyzu}&Y% zv#-AT>g!)W`KPB}|K{soKYjZ3lV5-RPfwmb`Rc2u|3scVz_$lSrAk`-(~~DOm$JB{ zz0awVE|&o~PjI;Zej($R&ZSe(`#HLGD#Z#@o(4QTHBQX1%$qDaf4h4!|IM@c6Wi;z z>zP?nr01ryE*GnLA`2#!{>oo=Fz*+$+3Xi}=)rSBbIv}NOhDR$Yb5;i7q?ub&&i8Q zm2&gmP&3G+1;f!VtPxd{G%AF|U>-y65_GO+teCBsm}|LZQSmHg1pv37zI!r%`qdoM zkL_CK^^6x;mHs3r!>SbXJ9|^5>-cA%thAO-r_dHbnTpXQX4Z;V-y3 zE$KqbWPfh`;SDzmYo1jF&6*QOX0et9R3&eG6#vA*2Lzx4}(HAJ7+MI|-&w0SEU z#*EMu4T?0oDm1e$UdXK42=|2@JA;)8mwgEBDQ2ovzUKq- z!2CBkZmqS3Zu+VmePIX!)PPGc6URg=VklGyNV7x-@?QzaO*Wbt%W?NBcfs^f zJ5`qxZay^GfH2j;kO=TVZjyKP?q=C)gp@TF&OzvF0ABP^95}3};2sz&P2e8oW(*i? zV0&!qK*6nn-tLm*PT}5uug}cF2h>xzkdVgWQ$+$!;J9d(481f%Xr&!V1v-vCwPIjK zu=Zb<2M|s!QMh7aHj4D%?gE^&o&r8Pdi)2VC;95kowat7yRd|pp)MkRar`qLhM2%+ zuYl2JtIXs|G`*{Ex8|2+@NZwOdp>!?9~PV%cpXFkl?3R+GvcR4jej{Co+i@XL!I4% z=ic)CxwfI@tkL#W9VfSVjG%Ra25~1`C}p_E_EZWeq;u!nEG=0*=YA$5ntbWpYVd!k z$z-7AkhMxQiF+nT+s#Gk(|Px&&Dw-d0uT`@vmTg3nO6mJNzVBkXe*!l{H1!n-eJH0 zn3xMdA?6n(^WMQ$T#aC0mBrhg(JfWh!mL0|-?4XrR_yr4;QkZxM60hir%hrNXR8G* zH6d`R%y3$x8o}VbthLvaZluKQAyez6wwzE^sn1Q8fzy$@ZFd4kzlv-6tnaBe)EreF zL&mvhQDV@LtXWH0M1C3WgE1%gU^gVQ8}ZF(GDvsU!X_G{Eh!GQTccaeMyYq82NL~! zeFHvl27y)!;s{Juo+hB-C6=ds7TT0gXREw0IXxAA3t(2wBc!-q9RG z*@KsVK6^_{yC*^~-#x3a6kB-HN#hTTumM~eox*eX?P{)F1rsx3y?DFXMtlyI@)Bc> zv5MG4A%WW~X~Sk$tlIRE^~eGOH$nQcvyp}@+PIaHHq3e_D3(SpTS)i;vWV1htk>I8 z$IegUiv%^5<;)3{MpRkfrq%$n)FPH9)2*fZ5{5v_{sBBnwX>d1`lbn^FalhJl8mE{ zRZJD*dtDwS1?TE#t;e7NWj_ji06_quoZLKbRPq_EnLl>kv)U;UMYyqL7+zcv#gG_bPI?z|N0~RHsR^WPJ zL1S)D-OA0jSiocU&04RDW$@HCu9I5sV}pL@Xw->>e+Vo?#?v9HD1yX0 zqQvVOZCab48g0>yMZu=A3oA7W5>bpi0hqP=sx&?VaNAMbF5*~#oToKxd}f+|GMa*i za=u|mLsI*)V3HWA74_;yhC-9jvLkG5$xunc>$m&8u7 z0DWWy?lzRHtc>={=3OG8YCQoZ7F)#HM4sDtW}$V=bdLm_i&ciVl7$|g2V`Ip`p16S zujS_$1|4blAFhzt+*0$hJx`Z${>rF8=;(}ap!Iv$H29Z!VkV9vkATAL&*>e9^%u>3r*oa=lWWnIy)CwBWjOl8*ngi0 zcFpF|e$r;U?Wh*oBP;*Bo)c;4B)d~&dy>ii&T_q*c&(T`T;VtnS7mg8#|psdzUE0d z5zQe&to!JL6|H;^`H|>6+OK7!b)ageGUJ?M4~`_V;BwMchyHx{_9r(?WWKf_5q2ZL<>m*wYdxO53I9#LA30E;X^{-aAI^i1cQ#^zCBl#)$)Z%p0T1F=G|m!bic43_Za9c(PCr`m$A4w9#guwMnXNkW@j_)hO@BIuOiDtm ztc+ICE_TJs|InC-d)Yu;rMqPnq&EhIouP>CE{=4ids}p>uxR5D{^V>guLwXE!0#b2 z*Z=H22FLE0dSgBQgyjCkc9-S$^ZaRsvfN)Ki)|--=TFJ<0_RV7hXi`%`G2vK;f_X0 zLBW5T82!&bFqL6iAO(?SS}>QK9|89y^%t9IG0N(`)8q=beoDY|Xcf4=L-tiVc-#zo zEJ*0k6~wCbEB3mdwH6r5A+`voxjR3Rpw!SJK?jH`-FZm1N}YZY#ccxxmj8Ip_4YnR zgync{_;{{{XES0A7}Hip*-#u{FC8nP#5(XP09qU3UOP=ORWW~NodDCgYxC5Y zy-E;=O}s z8ZhHwrte> zN<3mg9&QEdSNSvDOk@y>Xv;X9pWC1+SMy;DIss6=0W4T|n2(}F&$u$YHJEq^e$CMcf zh5^oMFnJSSZGUY;g%LXrbc2aMUs+Sc4}C}eb|~mfr+D9WWh-SYSV#I-)RjpH@#o>Q zrw%HXT|9BIt+HviUUW>GT;dNALhu|;Bx1B)`|hVXugBJ*tOprK4x=<4PL@s8e-D~5FxIk~;b}DP_ zOg;}z>`fd5ai3kcJ=xiUD;0iW$kxT`rMI@_n;=f4lSRr?8uXwT)pHbe9&^nEx(7^I zAVyx^^;NEvm(DS|j;;8OexPHhAi|v82clMPimll!nv%8hRYe_c)+#CTP(w&;E1mghGM z7s*M7YfnEt0b$NiX&*TvL((o&r6Qz&qkMDod+_z;no-N88!B4%qA?Qxh zqw)R+OJj(N%&QVjM2l($Ar?h|!DugRXzJY=WvM`Z5FMepnC$YG6dT4nWU;ZdDLnN3W zV;{srcYs~%6IJZD(CmBEC>|2V5!9d%V^HGIak2|@+S0H z80tzQ-jY1A@oATYlm?E~_w4%|;|dk;hii{3o8V+p@W)Se66f)L@py>&K_+#&;SDx9 zozLN(N5FGk64hxW$>o~&5BdsgL(=?`qUMbKElZFq1Xj3*)cPg?EfJ8S_@rcXR_=T}|wEFokK&K>;?3Mm>DKXc`kQj-UmI(uJ`I+I7A6&33PqTWc2Wr1p< z(XpUJu|Pbd|ASXn8Y0MJWfz%-Ap93=ZexC(Gd?+rg&G#%X$289YQDGB&260Lw9YX- zl|{J(4}ozl4hcU9XTFV_q{Pwzca@F63Ja7~_o(r$#Sl=XF#I4T^@+>%dW>R?)DRAapQ6G;h0LOKD<)a_`HD%bzJ= zB#t%6?hdHro8KEt%n;EGao68#`R3C9- zC2H)bi!G>YKN>RpD#6-@M7G;E5F3Bbr)TedB7aM&Af=yL<_CzB=ioyO82xg<2F$I4 zQ2;d?8R0Je^eT_JmC!SSC^@|<+1&e2`AbXC78jvS0Oj!|XbRS{0~0(GYJPS!g6!yc zxe>s@sH$&`h3p96N6~Qm(-R9Yp%LIUI{M3_`doyv%wNA`aPQ0JI+n?ECbL*Oi|=RTC?&H{cMtw|_4!$K_WnCD zk-hid<%j9e#a;Zz^N;zDtN%Ft&m*$94MOuivRs8Ojy<2-MWASx04ODzhoA+Udov2% zIeS!o_3xbDvlLOt1KAeq%v9uY0vEWosRQmnP@X>I|GqF#|9N@t-j0~p zu#zVYow1(cD=B7{PyqjN{9maY-#}d>3jLr>D8y@{#foFhg&0=*s<-dqG+ zuK@v4Q}eNGr;CrZ%&Pt_FMv!rvQguxDgj5mR;%g+h5Vti`}A3H2gDzM>tx0`;JWo;t2~PYUiV{sH5s+*55#wv z@3SnfL=f9)#~?>EJPZOd2*B}U*iy;BWilzb@K!RgeZp{O*TVPI6Kv8@YqlDU09co~ zTL@Sd&n1Sh&rTQraz)=o0GiMw*C9M|9#ch$8H0^B<;RT)*<3^p)nl$Z;3TRnPI=mz zZM0Q807^2|e85>Le&3>|4LZyp{|w60qd%dpd8NMcD`uJ1r>CbaSTl9Bpz!)+zlbp`CH~qVab2n_wgb7^oUqa&Zb4W!7MeiR?H=SDmPJw zeu2}h@u#jip1^z7EN&%i{?i}+;ZR*Bre3t?>%`lNs7ZCc70ZT*#o6=;21D80c5aFw z|Aw1)SD^@PD8%G!E0KYN?HmzE1fE?Uj020i>0f-0hf&;SD|Yxln9=F(B<$-6_l4d+ zfRMMypcS>rIwjM@2E=4nSbVYjyIJ}FVPNAsrsU^6IYFVuPb*2N zG&Fnqf0p`)CI180OVjTb>*KwM&(}l+Y#TRzneO|E1bg=VWcC+C>t_)wJ5h)U$!*Quji1thVT zSR}DHR=SoktoCOVtw%GS`_;|#& z`{DWd`SJPSw`zym3q@A2o9{0!;QfAVMY;azsCEo$zRVpjjo{>@6Ehzo8CTH463`#$Y##`%PMin& zw19ng+TNQZI|A|i`Ktybvfez|kdW{j9+K<6tbD#k#piX`s-Po-YHlgwdDQ#qzk=t- z^V5KA{#=ApgfH`CI~v&f2w+wo^|etc;SyYICC7^$=fs}!T?1fZ8)z&&+Wt57_nV=l zJP4t+ESMQWUh7zJaR0mSr_X8w`PrG7x?2DDlfMxLKSjr@`Lv)MW@88iO50F$ZmJxu zfC~G!Lyt0hAUjPkSEy#1`+lCd11nLvpc^eidIQ$>w!C%Hj2`q@i(pz%{EuAlo)8UR z82dmV;CzdW?cuPL?6!+x&2;8-p8RCAvY+!&KwXB#kt&>Sqt_`3pNPmy#KQs2>k2yq z`yMV20)3_VDzLk+oBzwr-;KkGFDLXd&MWb>>)1rIcv>D-@ie|C6z$0avpS{%DTJv< za*q;13{5KHHQ1-Rp>hI*Uje?{@1L{z_;zu<75=@? z{_;Kf8auNcuT4(@4Dr33A4i4MofAh`73=q1?;kLk8ac;?kzQ=vQwYC|<1DbrDZhqC zn5$y+=FU1SsS)KRt}n#7aDZ4cVI}KQr^-3SCMNILJnBi zv_xHb#3=(cO<_Eix$%x}rTnb~s*LaAgbc*$)lHhE1bsJJ0Tz=(YaAmj`0ouWbG-s= zjfonV#qaRX0Qu|Q_2B;|x^9Bo0{ zUBCSjSKThL1ATeXYfZ#l8D`8p<)#8`qbL;acB_rqxYdCgu zEYnhfM@%8aG}^aDI5wxo?eSd7q=Q`_)S*iFp{7%1mFxJI%2wnyl-)PjjqfeNXY{Nr z;aAaN{J{CM+QUWOTZtCA&FO6qSueByr@NeJ`0wDx<8&+qdu`5sb++P_Hrt7$rPDu_ z!7M3v-Th@tNi4rnz-WKQ$Ue;wX!* z@0BRi_pVfzWJoNx>nd!QvkaVkf!_)ci)_A6_@E)jN6vcp?h+##R8I4vxmUrBO0x1w zE<-DpX;h{r)+elzwbqw5Q+Cs?=%aYf@rv8uYhwQ>TUC=b9_Gr!$Cb1QuOrr2b3Z6b zi>w!>u%VR~9hZ-hC2ZrY$pe;yl1J8Oc3Vg)#G{nJsQpIMd9CiC((jaoylz*Ru6qR} zkTvCUFD6o;qyZI6dv7;oV4z-J5MBFF9imYjL5yrlAlu#&Er>si)MI`!!Pbk2twJ@h zI`1Bc&hHCTLS-1)Mj{Cug3Z>Ctuxq)M`gYuG4^C{Ilx=mI+KLpb!`?oY%op8s9BE& zkL8+6xo%qkML4(I_DKB%tR0Xz7Y)F!dXM`)bP(X^|C+)2JO|=xKhU(Z06#&azC|-! zw%B#rO zu4M&xirUxz!{oxk03Q%0o={9eq9u>ssuavE7x3uE({!q;qF$!I0(a!BN?A=H)*n5C zY!$Zfy&{&6S0AyNd}?l^*8f$xi^ds&mX24uF?tf9qK7xf?wAEbTxxY~i!BD4Uz3BE z+c`K=l<59q@`)g!21yw7_U5*kPx zuzQ8J`R!2+;9CBnNVAOi@Y_CTD(oE!@fE-)Zpa79_X&TEF+J_ezS^S5IsNq?KAnih zBYY|WdrJ7`jf1mF`&1-Rf7iB~Zo$AJHE zdM$w~3Aib#AzT$7Gb=MWcCsFWM2Fq~RfeQ)oNO!z4THhsCY?J+&kfWsAvv}X;I>5d zVIFf9Lm!vcyz!e${#!jhxU)0K42z?E3whDa!x?nXd;(62>IVnDB3720M^oAh^{?YF z-H4^MGn+t@fW+oNn4@E8LPZO%VPgrSG%Iy5^w(dloQeleLH*RKXO)@MS^wo~}5!P7G-wiKyjB4hE^i)K|uCM3g ze)X?af*FCSxHAk11K)M!{0<8X>P4%_B{gxfQr=vzq?J`8>I{#D3jMbHjF20 ztqn;uwjf&0enCVDW~W(gfw@j}Q8|aE$6EbQx{FKoG>D+6>e|j7#4om$0=wC$pbiYILT!r1olJ2tYKF5*UgE-cU^KLx}H@O;MW7{8FX^o$2m23S*(c5qC#40XP+dJkJRp@@I&QSS@1JuIY>Xiutu>c znu%m`w#-gJYK_5TO*ZLvYrXEblybyuqQ>X%s*8Z&ZovTYTN_{hx_?%4{=|}_PST<{ z8ec5lb?O;TT#2k?(CD|%F*~(Uh^PpQpWMsHu|>hAIKwhzU71y~p1>O`*At^x#~JF5 zg!MDzUm2n6v2GSpJLIjlp3o0cg@kB}qNiHERNxc$5YylT98flnI4IZ^Y%KcJ!|g>< zI(H_gX!fl?FfZg|guyH>XS#<_Qxdbc=qzSU@j6&Nc?e*L=-n-C(vqnUXOVE3&&hY}l^jcbjY#nvk+RwqVRf&p&CG-C1rRQja*DvKa z7OAGGg37xGFO|e&X{XB1%jjP$`#(U}2qrYk(rUT3KSY`IgbRBNE>GYkLep<;^w^0w z525z@1f8YG&L=B$3-F!i+YzRNTyWe6vgSgj|Bbc%&t=A$fCb zHQLAuHfx;uA&F;2_^DSqd9_%E-%48)BRRWKG3|38GI1W!=&ewY8XVh{ek6< za!I=51<>Xi`tXpWW##^rN{LZw&Dp!4njrTi)*zkiw588o8j4wqnTm_Xh7twIkl88~ zsJm$o;uK5QC32QXQf6t1{^-x@jD@;B0?swi@B1`*c{zCCmW1aPf~*jQV>=ryk*7u~ zgUA)^`k&!%duk*x_)d#WA+m!?A&N6SYNi-$SBG#3G6O%Q zPAl0@wF=evox{UrOuBJQTgDncsT=?F) zLc4DjjR!VE2Ef@>`tS79^i_;H`kCG95j&8y5hDL&B3GXJoUAI0V9RwqbDUGeGeKGG zD$SLC4Hb__JL+OHnmI+w5kUxJ2RWcK|Mf_{PrghkAX~{hTO%sq$oYeEfaYLr36XGx z83S-Z-1{0biXcWl@A){GI4Hvo&A=m39EW~%$qR6c(PI!C#yZNrLOllEBPHJmS=({tJz0Wg8#`-fC->4f2s=2T1lvDo{ z<%0VWLCF01`o^vJ`Pn_9ZMg8|qqw>;JdJoCMYfxWr2~j7EcploYhhF8{_9+7&yPVZ z29rB(3K$zdlPZQmr^o!g1upU1Y6EjmdFJ=SfdPSDu7CUakdOi%CybxP{sa}>4FVj2 zfso!4p35&4SCJx-$4M*Y6sFa!9~hFH-_s4B6jbV@t5yA?)bw{rxEchJ;JSPc zf*j->umUBQgqIPL(uKz*wkjCKn>=<^@}y@|)As9=XW=?%n6Y9qj? zVy!DMV$~?V9&n@*RC+4o#dC;O$6t2CWP*W?Lu^Sta-KyG^ z29Aj)X197rFUeiOGKzlJxtj;@vz;>7A44_@-;3o06w9Wis7O& z8@_Ya&+F7vL=Kn6{spRnKNPJ4+K)G9Y#9AU#w0yfc>-L)r<>y$Dz&~2P*-_!Tg@;A z${i9j8WMRA$$`IBUc73_<#}dd+=%GC;V$Y&HVC<9p=tjN%B^JJq0vKmLDliiiRHD zXol8PcN^>`tP=jftkQfIrC)i(u7)-=s{9aYjNu3zFkY`}|IX+W=Fl009!yF~MoXmf zlpC&fTT0dIKC=SBE^cMDwOaBzqq=L`?@#;sM_lBx`L4UYpO5BP6&BKOwG|2MOR&%G zI0_1ejVGtCb%dSkJI|NYMA{wjmZ8#UlLMv1OmNn#(m>E-D?&6D{4$JPm`kWtM@<@L zn*|o{#@3Yk-A%aQe%jwi^y$CKou@@_VX;u$)@A-`3 z|JeymM3vHNQJgt%HWr!=bPQQ+lbhiPqS)E+&{=yZYbIBzOAM<$U&;@NsgW`lgUGf@ zd>v1X=E=@g?`)UKT#L#Fw^H7%h%rglBlb*dVHebe(kD zj43OJjj2PlYNpu9^Sm+DuGOYaU#JIHW`qy*kGY8}C6^#uHO*5QOjA*FP;^?;^5_Y8Fg8s1mc$%q&7^voc)}{*`5hhj9qm=y@ z*&%PtJ`~GD?O$@p^n?b|PLbpF0aXrg_0dpO!7(sPIce7*TFhcBIs}2cf+&-4cNlX& z*_CS2l4BxH`M zPcVI9Deg+xgIM5ScY8wvOvKb&!fiQtlxs6ooj;hkRq=M7QJ7IJ2^Yn5%vp-5c7bt3 z`qaru5vkKdLZ=?8>ya6h6}~}vAmmtC;q{_eddbOX1DV=b7MzE+4+O-XS37A_y?BR_ zP-`K7ndY$Zkr~?ZV-UmGT0X4}BR=cu>MRV9TEx^@NiocTtm%+o=^y+S(r+Ms`86FY zXNH;%EoZKXp*e6S7mo`LsJJp^7ZG??TJQrNZB>q2q>JWIzB(xOGkm)UOQY+c<%xa@ zOEV)!LnNE;dzo?=)0OXW|6HMFQ!&;WOD##W&9t}s2*J3p(QWY=RN?8eRo6N1l%~b} z4HM%IMy(SW%G62_50c#7WOSg#KFSxhd%$=Y$g(I;V=3dAXPP#|~N#$hPZF zMhwffo|ZVY;2&$^47CudkXIEnw~-Uo$sJ#*3cg(5<+l3y&XG{M^EaE0Jrn%1Z=@X( z(yZ-#8)_%33TX$c=3tsr({i}m%}SV?INJa-mFBeeEIYeb7w|r55s;Z>#Vwgp_8g~H)u|f&PBIMGz7hJ#SgCdDZe0c)gj z!dH&Bn!W-OlWJg8$TVgW%oc?E_55gQ`kO@me$8B?jUiM;%4sAyJE%%t56eEfcGess zm_KT=ZPg9a)iJ(4jmv=Nb8dB|vy5#XN4bL9%PB57TZdM;UfW3Ql(o1pMZ{EE8484H zBgQ8+3f`#h>SMIT_!P0w<{pP8pN?_5pI4h%g&itbEUAbYdzIpetKO=)HCjl)5-kx8@I-Rsk zdv%78bV?Js(=H8Sr0Jqa+W5r!7iQuE*k3p^qbIili&8>>_c3XN8XiC_`;rXFT%VAd zX;dd3hPk$iZv8>wX$VcT7E0Bf$U{{M)N^z(B*fc?Rm=b;2T>JK$V*@~$aFYmmzkA9 zW}<`#U1eb*hk#SEqDGqV*&x^>76gR7jb#5hw(ZlA*KoUEe~(ut$1EEgMOU-LdOKV{ z;~TwjC}|)@C22o43Q<=PA`Olx9)JC~Bp&5^g(0Ix@)RkgEojg5nep}NthfqrR*dbZ zS{(m_K@!VfH80V^CQh(?R=lCL{a?j5=6{qIB#qRu{L}$4{8ed89~J+#__=HP9x|Ey zXK=eFf#y5RleyFp*tr!v>C%vwzp=WM#k7&nxxNSn?pU2eh^XQPT2hGEAz=@+@XZ(AJfzusN7qkvO&rF4#za$Zad-J zf0C9U4Yg?cB~OYUifz<%mlY7kNv{mo(`nrL=O+@}vs ztigNPV-oxM9L}7T-XX40S%<8r@$IfPo*)?WL1t8MVok_Orr3+L7*oRD z)gs$gq+-}_xK+M2Ly10!BIj1(@oApRKJJuIRpP;uk}2;%}q`00JzZbd#rr0sb*cF7lm!XP;ljqP3X2Cait9doH*)4xx_-lgO`+Y~s=g z@%tv-eF07x4)=L2T(3tezHk8tJAP51(?p=Yk)IC){Mu%Fl=((AeU5>3j4KM5ha_8d z0gT?&CZgSxG&_v2JA{|ph602zp`k_B3mDO zuk+&$b72G)3{v478b;-->Qo(guW9o#+4BQ5TeE1nJ#(GUIrH<(s*b4{TBGhSqKRsy zug1)xJ{oayV~=w~Z&|yU&~q=F z{6~xoM)L+>j6)Y27tn^TJQIc<`ZT+v9A&?ON3snR5bu;7dXYJantYsyJlHM_^Ruwj zB4?nWhojGQlu8AKi06u?8e``LaK>%ODM0bh`w-Rel}J#>v-RjYQG;<1=3(bw-d;Y^ z0xO82RiHSSGfDL>=;l%Re&95W+5biX9f4y&=UU)24{E7#;1_2n@H_Aeh-rvoJ(e2n zV_C>(GO`npcz{c>Zjq8?q>6dwWf|CcQf={;rv^3OV8}EbjIs<*I8ND)ZIW8rW!FQRZvR$DdyMzY&1rZA{5J#lv<#%=8vmXDUaK(#6!^nLln9K86rzGWc%F&#=3M#$mJO5Ul@#IIL-c(nG7(?k3iIw#gc` zI+`4LB^(pJ-5+m0(m+knf*tW@XMnx&_yU9Y@C0qRd#boju%D2*q*jPxrA7m(w5Exi zCiy(L!u-2*uiIxu(;lv2X1;WBtpPw4y5@P|3`7P?#>%%lcv^b^WgxA|!k?x|8;Yf| z3UMLH1jE!Mq5T}tkm*J?o*^wiP1FynJVnE%botcQrJ-}wZfqAmgTDrpD5w2YM)`cS zod%pIh7goO%c7}5+_*4k<97M6omMj_|7u7`p<^MS=%S-(h&8;hg13B8RPiovU9%bL znQ=cOK0;{9uzYtbby?GQ6vMM}jL9nsDByYerD*)U!O$`s0Wo~xQ&#yH`1P*8>->e@ znGqxwh8={N$RietZhzH#U8)!HJ(u@uuEbN976ft)3O15=5J=LcU%UG|OUBKLL89r! z@9U4QN)piU==S1ZuqxYuOp0@)BV&U(4YuRR4ME4C|Lo|v9&M+GpN!ToY%yg(anAw{ zxI%e&&t+68o7O@AwBXg=H~UyeqY%jGIZhH-*U_}yA-WL=@F@<4G74weShfmOt>+-< zA9-3EKcwbGHauT+_L?rn3X5MCo9Rs^@Nk`+y18L~RK1jck3MYC&l2!kJ~Gh#;J@dl zO9*zz%qE>nN^#B>oJH7k;b`r>lP?mA1#HQ)58b)3xM_~Ci?$Z{I_p zUZ&B~c&BssnEyiWSc@>uM!vt# zI(GTR*5?qMc&qr&)vwkKQGS#%q#TxDC>7&k1+&pb8b8L^a?sih&wPuVK9^M{WxyfF zpj=g^#M?P#$lW7)jZ3%fsEQn*u-Z&Dx%r|ycE!1{+!K@&Q@&m4H%5xKPC2JVIwPUEAHKE+p}7ZB zgNk$o{;|5Rknf*?>*wV8cDOjc8}R*fS5Ec!?zEU8j=w#?*W1X|YPG;qNu;rv-oD|+V#@!044017KG7sGK&w+9b zG5J8Lz00k!MfYg3znBj%?n38naE{{1ws3#>F)b^(%;8o!C`Nh8r>^8SQ|>b*`KxTC z)f(NRmv|bQWpQih$=INk?&j^sc(#IaO#g<BX&+1}sJljasu*NV-$R6I&=mX} z@vezuJvR}==Spn>)t9ue8}o5Z+uw`79A)<$fY8?K1R`O~MBd13%RP7KAlYr!nAg;< zlIyafe#;lwm%3?&^7}DP(3D%V&LXVh%ym_MOrLzjHsC4~JriK)I@EkaY0*j+=|BCh zv#XDJ)KN0uC|m83;0%Fy2x>j#oJ=5`WFG2$Ft*TqMH6wzM32xRxCw<9)P;R|!qndt zhrCN+rC0H^+3uu)?Hu`-xQkNm)U-TH_#SN^Yj}5VW7eA;GbhRngcle`wKn9P7xoVx zNycRQ$Erf3Pyffr?((SwWH(5^-8HZ3ge2Fol)RmmXmn;cszabHqe@Sv*eM{JrQV6x zuT?@Ww?~mi{AOvTrtEAml#75PAL`bwMRPXsK-Z`1bJ67&ZL~;M4RGN4Xqagr3fQ0e zCt{&Sx!8zY@v(v>5-DYvDt3z;D(BVnVF%@}>Ju6G&RZa2{S4h^eu5);ihi!37*4+s z!Gr|SOcLNKn5z2~0FMz5M$SF^cUOr`4pJGfB#YzITR;9rtE7>aii;~Vm8Uk#9ZDP9 z9gTB2z9}{63mb~797`kVDf0l9B;tK-(INtrBXzKaG7B3W>oe@d3?8g|DOuT63OyCB zp_Om1wm(h%9L3(>wjRfzk7=%1TWN}duH1CFXS2T;;`Raa21%BQ`(~VLTUjlwn>ow$ z$F*|M?9C?8w)2{Uc^9 zW95?qXP64ry9=?1zS29M{4*LvuK_dTKmT`4*%0Es;7h&h#A|~;X0hMtiDxqELLiQx zK(_`v7h!UdUrr^7{e|L)xl=-UDiidUi*WwxdU@m@&V!k0qH_Sm_?s+fY-Y&IbM7BK z#+$35Wf_#E<2d!2wK!>;5@?zv@&TXt{=W#eV6Ex@* z^qTuq>BVkuXNp8>P(fSOXXlqS4rZr)k!kq@&A_BSLNke>pWD6iVN{x+5RtL8ZGrQQ zeAHBe!!p#w?gdVx8^r}D*As*T;tXkgqST*k4;{3okNoe)oNBe^gNT|h8;p|%j94v5 zfFe6oGf2KQk)*KlU{V2`W}#>wBPXuj3yvd{e^~t}TQZR~+WzP>7 zKH~aH5CU4VHDmMw`OL+SvYceuTf%APRn1&5&}^%?bTA_|ql$w_MlBd0#opL)`TNb^ z={prF7|&3FTr~{1j@@?G zSl2SF^X3N2o_$#D7yDUmR||pl`yWzc#D>4_v%?D*=d5uYmGMF=Ig~lZ%K<8IL4UBn zesSIuP;UsCf5vrTm}gA6CoBYF;}bXa@lmB7v6&s`O37oHLv(OQXeb@HL&>j*7M0Cp zAz{V1%=f1rA~>O+!Ll2flM0rDn^VntkXM2mEl?)$*&!BTF5!R&0$4!IOyxk#1lnlA zE;*iPx0FDkQuevVZDmWK;}Ei;APTP%&Y1GxCkF!nEXEH?TScsq@z!#8dkJh-Yz-ac zibVotY^lIIT}|ZEHQ6>GaqssvR1=b9Qb~8+8PK(A(?Y!cz1lo;vfGhC0{}su?7;fF zq`7)#(PguuuYAc&2+9F=7AFJCw=GuH&xr3z@uXQhTmmvEe zhR0K^S**YG9{nlYA9yl4RZw8`pjX zq0RU}hY!~+1aEQcqoT2ydzrg269P;$RfvELN(V|AyS)59hJjRdKZY1ZF0oyqckg(~ zZb1PiRXhAFQQUWQ@(4yYsCmN&9+F~i(Cw04VjQpw@(w?vUbDb(?;Vss!5sOp!Ot%b z8h^?$9p^rJc^xd3Yi|S&FY^nU*z#;$M_b&I!88~TbxtXu0o`c%<&mP{a zQgph87wEk4XWi7zAhOU&Ux(n)yF|@~>W3F`c2R#!CKu;E{NSOG^>)H4w52Yrrslz{T0R7 zvZjidwQ2YvepbALQ@+ku9UV|K2U5=`~Za7Zl7B8~i>3b>~tSQs!>+ zK|e!Suxejztt!)`#jv8nh(cH%VrN3?a4_?)e`m#Wl5bkaAk;dPzw1>CX$vf{N&_f~ z<%-?L81XjDRyrE`JIvXU*Xi<-$-)Ey@^fea&O2q$xL8(6w25